From 277b7e11dd23538fda92bce826b7db5eeaec0ec8 Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Fri, 9 Aug 2024 08:56:01 -0700 Subject: [PATCH 1/2] checkpoint Signed-off-by: Jeff Ortel --- api/analysis.go | 52 ++++++--------- binding/application.go | 60 +++++++++-------- binding/client.go | 142 +++++++++++++++++++++++------------------ hack/add/analysis.sh | 77 +++++++++++++++++----- 4 files changed, 191 insertions(+), 140 deletions(-) diff --git a/api/analysis.go b/api/analysis.go index eb115113e..2a3e9c19b 100644 --- a/api/analysis.go +++ b/api/analysis.go @@ -320,7 +320,7 @@ func (h AnalysisHandler) AppList(ctx *gin.Context) { // @description - dependencies: file that multiple api.TechDependency resources. // @tags analyses // @produce json -// @success 201 {object} api.Analysis +// @success 201 {object} api.AnalysisManifest // @router /application/{id}/analyses [post] // @param id path int true "Application ID" func (h AnalysisHandler) AppCreate(ctx *gin.Context) { @@ -339,35 +339,14 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { } // // Analysis - input, err := ctx.FormFile(FileField) + manifest := &AnalysisManifest{} + err := h.Bind(ctx, manifest) if err != nil { - err = &BadRequestError{err.Error()} - _ = ctx.Error(err) - return - } - reader, err := input.Open() - if err != nil { - err = &BadRequestError{err.Error()} - _ = ctx.Error(err) - return - } - defer func() { - _ = reader.Close() - }() - encoding := input.Header.Get(ContentType) - d, err := h.Decoder(ctx, encoding, reader) - if err != nil { - err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } r := Analysis{} - err = d.Decode(&r) - if err != nil { - err = &BadRequestError{err.Error()} - _ = ctx.Error(err) - return - } + r.Commit = manifest.Commit analysis := r.Model() analysis.ApplicationID = id analysis.CreateUser = h.BaseHandler.CurrentUser(ctx) @@ -380,13 +359,14 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { } // // Issues - input, err = ctx.FormFile(IssueField) + file := &model.File{} + err = db.First(file, manifest.Issues.ID).Error if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } - reader, err = input.Open() + reader, err := os.Open(file.Path) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) @@ -395,8 +375,8 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { defer func() { _ = reader.Close() }() - encoding = input.Header.Get(ContentType) - d, err = h.Decoder(ctx, encoding, reader) + encoding := ctx.Request.Header.Get(ContentType) + d, err := h.Decoder(ctx, encoding, reader) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) @@ -425,13 +405,14 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { } // // Dependencies - input, err = ctx.FormFile(DepField) + file = &model.File{} + err = db.First(file, manifest.Dependencies.ID).Error if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } - reader, err = input.Open() + reader, err = os.Open(file.Path) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) @@ -440,7 +421,7 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { defer func() { _ = reader.Close() }() - encoding = input.Header.Get(ContentType) + encoding = ctx.Request.Header.Get(ContentType) d, err = h.Decoder(ctx, encoding, reader) if err != nil { err = &BadRequestError{err.Error()} @@ -2371,6 +2352,13 @@ type DepAppReport struct { } `json:"dependency"` } +// AnalysisManifest resource. +type AnalysisManifest struct { + Commit string `json:"commit,omitempty" yaml:",omitempty"` + Issues Ref `json:"issues"` + Dependencies Ref `json:"dependencies"` +} + // FactMap map. type FactMap map[string]any diff --git a/binding/application.go b/binding/application.go index b8b02fd62..fab15516e 100644 --- a/binding/application.go +++ b/binding/application.go @@ -1,16 +1,12 @@ package binding import ( - "bytes" "errors" - "io" - "net/http" "strconv" - mime "github.com/gin-gonic/gin/binding" + "github.com/gin-gonic/gin/binding" liberr "github.com/jortel/go-utils/error" "github.com/konveyor/tackle2-hub/api" - "gopkg.in/yaml.v2" ) // Application API. @@ -317,29 +313,37 @@ type Analysis struct { } // Create an analysis report. -func (h *Analysis) Create(r *api.Analysis, encoding string, issues, deps io.Reader) (err error) { +// The encoding must be: +// - application/json +// - application/x-yaml +// The `issues` is the path to a file containing api.Issue(s) +// as individual documents. +// The `deps` is the path to a file containing api.TechDependency(s) +// as individual documents. +func (h *Analysis) Create(commit, encoding, issues, deps string) (err error) { + switch encoding { + case "": + encoding = binding.MIMEJSON + case binding.MIMEJSON, + binding.MIMEYAML: + default: + err = liberr.New( + "Encoding: %s not supported", + encoding) + } + r := api.AnalysisManifest{Commit: commit} + file := File{client: h.client} + f, err := file.Post(issues) + if err != nil { + return + } + r.Issues = api.Ref{ID: f.ID} + f, err = file.Post(deps) + if err != nil { + return + } + r.Dependencies = api.Ref{ID: f.ID} path := Path(api.AppAnalysesRoot).Inject(Params{api.ID: h.appId}) - b, _ := yaml.Marshal(r) - err = h.client.FileSend( - path, - http.MethodPost, - []Field{ - { - Name: api.FileField, - Reader: bytes.NewReader(b), - Encoding: mime.MIMEYAML, - }, - { - Name: api.IssueField, - Encoding: encoding, - Reader: issues, - }, - { - Name: api.DepField, - Encoding: encoding, - Reader: deps, - }, - }, - r) + err = h.client.Encoding(encoding).Post(path, r) return } diff --git a/binding/client.go b/binding/client.go index 78ae07b85..93fa0140b 100644 --- a/binding/client.go +++ b/binding/client.go @@ -21,6 +21,7 @@ import ( "github.com/konveyor/tackle2-hub/api" qf "github.com/konveyor/tackle2-hub/binding/filter" "github.com/konveyor/tackle2-hub/tar" + "gopkg.in/yaml.v2" ) const ( @@ -71,7 +72,8 @@ func (s Path) Inject(p Params) (out string) { // NewClient Constructs a new client func NewClient(baseURL string) (client *Client) { client = &Client{ - BaseURL: baseURL, + encoding: binding.MIMEJSON, + BaseURL: baseURL, } client.Retry = RetryLimit return @@ -79,6 +81,8 @@ func NewClient(baseURL string) (client *Client) { // Client provides a REST client. type Client struct { + // encoding + encoding string // transport transport http.RoundTripper // baseURL for the nub. @@ -96,6 +100,17 @@ func (r *Client) Reset() { r.Error = nil } +// Encoding returns a client with the specified encoding. +func (r *Client) Encoding(encoding string) (r2 *Client) { + r2 = &Client{ + encoding: encoding, + transport: r.transport, + Login: r.Login, + Retry: r.Retry, + } + return +} + // Get a resource. func (r *Client) Get(path string, object any, params ...Param) (err error) { request := func() (request *http.Request, err error) { @@ -104,7 +119,6 @@ func (r *Client) Get(path string, object any, params ...Param) (err error) { Method: http.MethodGet, URL: r.join(path), } - request.Header.Set(api.Accept, binding.MIMEJSON) if len(params) > 0 { q := request.URL.Query() for _, p := range params { @@ -124,17 +138,7 @@ func (r *Client) Get(path string, object any, params ...Param) (err error) { status := response.StatusCode switch status { case http.StatusOK: - var body []byte - body, err = io.ReadAll(response.Body) - if err != nil { - err = liberr.Wrap(err) - return - } - err = json.Unmarshal(body, object) - if err != nil { - err = liberr.Wrap(err) - return - } + err = r.decode(object, response) default: err = r.restError(response) } @@ -145,19 +149,16 @@ func (r *Client) Get(path string, object any, params ...Param) (err error) { // Post a resource. func (r *Client) Post(path string, object any) (err error) { request := func() (request *http.Request, err error) { - bfr, err := json.Marshal(object) + body, err := r.encode(object) if err != nil { - err = liberr.Wrap(err) return } - reader := bytes.NewReader(bfr) request = &http.Request{ Header: http.Header{}, Method: http.MethodPost, - Body: io.NopCloser(reader), + Body: io.NopCloser(body), URL: r.join(path), } - request.Header.Set(api.Accept, binding.MIMEJSON) return } response, err := r.send(request) @@ -170,17 +171,7 @@ func (r *Client) Post(path string, object any) (err error) { case http.StatusNoContent: case http.StatusOK, http.StatusCreated: - var body []byte - body, err = io.ReadAll(response.Body) - if err != nil { - err = liberr.Wrap(err) - return - } - err = json.Unmarshal(body, object) - if err != nil { - err = liberr.Wrap(err) - return - } + err = r.decode(object, response) default: err = r.restError(response) } @@ -190,19 +181,16 @@ func (r *Client) Post(path string, object any) (err error) { // Put a resource. func (r *Client) Put(path string, object any, params ...Param) (err error) { request := func() (request *http.Request, err error) { - bfr, err := json.Marshal(object) + body, err := r.encode(object) if err != nil { - err = liberr.Wrap(err) return } - reader := bytes.NewReader(bfr) request = &http.Request{ Header: http.Header{}, Method: http.MethodPut, - Body: io.NopCloser(reader), + Body: io.NopCloser(body), URL: r.join(path), } - request.Header.Set(api.Accept, binding.MIMEJSON) if len(params) > 0 { q := request.URL.Query() for _, p := range params { @@ -222,17 +210,7 @@ func (r *Client) Put(path string, object any, params ...Param) (err error) { case http.StatusNoContent: case http.StatusOK, http.StatusCreated: - var body []byte - body, err = io.ReadAll(response.Body) - if err != nil { - err = liberr.Wrap(err) - return - } - err = json.Unmarshal(body, object) - if err != nil { - err = liberr.Wrap(err) - return - } + err = r.decode(object, response) default: err = r.restError(response) } @@ -243,19 +221,16 @@ func (r *Client) Put(path string, object any, params ...Param) (err error) { // Patch a resource. func (r *Client) Patch(path string, object any, params ...Param) (err error) { request := func() (request *http.Request, err error) { - bfr, err := json.Marshal(object) + body, err := r.encode(object) if err != nil { - err = liberr.Wrap(err) return } - reader := bytes.NewReader(bfr) request = &http.Request{ Header: http.Header{}, Method: http.MethodPatch, - Body: io.NopCloser(reader), + Body: io.NopCloser(body), URL: r.join(path), } - request.Header.Set(api.Accept, binding.MIMEJSON) if len(params) > 0 { q := request.URL.Query() for _, p := range params { @@ -275,17 +250,7 @@ func (r *Client) Patch(path string, object any, params ...Param) (err error) { case http.StatusNoContent: case http.StatusOK, http.StatusCreated: - var body []byte - body, err = io.ReadAll(response.Body) - if err != nil { - err = liberr.Wrap(err) - return - } - err = json.Unmarshal(body, object) - if err != nil { - err = liberr.Wrap(err) - return - } + err = r.decode(object, response) default: err = r.restError(response) } @@ -301,7 +266,6 @@ func (r *Client) Delete(path string, params ...Param) (err error) { Method: http.MethodDelete, URL: r.join(path), } - request.Header.Set(api.Accept, binding.MIMEJSON) if len(params) > 0 { q := request.URL.Query() for _, p := range params { @@ -542,7 +506,6 @@ func (r *Client) FileSend(path, method string, fields []Field, object any) (err URL: r.join(path), } mp := multipart.NewWriter(pw) - request.Header.Set(api.Accept, binding.MIMEJSON) request.Header.Add(api.ContentType, mp.FormDataContentType()) go func() { var err error @@ -685,6 +648,24 @@ func (r *Client) send(rb func() (*http.Request, error)) (response *http.Response if err != nil { return } + switch r.encoding { + case "": + fallthrough + case binding.MIMEJSON, + binding.MIMEYAML: + h := request.Header + if h.Get(api.ContentType) == "" { + h.Set(api.ContentType, r.encoding) + } + if h.Get(api.Accept) == "" { + h.Set(api.Accept, r.encoding) + } + default: + err = liberr.New( + "Encoding: %s not supported.", + r.Encoding) + return + } request.Header.Set(api.Authorization, "Bearer "+r.Login.Token) client := http.Client{Transport: r.transport} response, err = client.Do(request) @@ -777,6 +758,41 @@ func (r *Client) restError(response *http.Response) (err error) { return } +// decode body into object. +func (r *Client) decode(object any, response *http.Response) (err error) { + encoding := response.Header.Get(api.ContentType) + switch encoding { + case "", binding.MIMEJSON: + d := json.NewDecoder(response.Body) + err = d.Decode(object) + err = liberr.Wrap(err) + case binding.MIMEYAML: + d := yaml.NewDecoder(response.Body) + err = d.Decode(object) + err = liberr.Wrap(err) + default: + err = liberr.New("Encoding: %s not supported.", encoding) + } + return +} + +// bind response. +func (r *Client) encode(object any) (reader io.Reader, err error) { + var b []byte + switch r.encoding { + case "", binding.MIMEJSON: + b, err = json.Marshal(object) + case binding.MIMEYAML: + b, err = yaml.Marshal(object) + default: + err = liberr.New("Encoding: %s not supported.", r.Encoding) + } + if err == nil { + reader = bytes.NewReader(b) + } + return +} + // Field file upload form field. type Field struct { Name string diff --git a/hack/add/analysis.sh b/hack/add/analysis.sh index bb2cb79e9..af1658af1 100755 --- a/hack/add/analysis.sh +++ b/hack/add/analysis.sh @@ -3,15 +3,15 @@ set -e host="${HOST:-localhost:8080}" -app="${1:-1}" +appId="${1:-1}" nRuleSet="${2:-10}" nIssue="${3:-10}" nIncident="${4:-25}" -aPath="/tmp/analysis.yaml" +tmp=/tmp/${self}-${pid} iPath="/tmp/issues.yaml" dPath="/tmp/deps.yaml" -echo " Application: ${app}" +echo " Application: ${appId}" echo " RuleSets: ${nRuleSet}" echo " Issues: ${nIssue}" echo " Incidents: ${nIncident}" @@ -192,23 +192,66 @@ echo -n "--- name: github.com/java version: 8 " >> ${file} + +echo "Report (files) GENERATED" + # -# Analysis +# Post issues. +code=$(curl -kSs -o ${tmp} -w "%{http_code}" -F "file=@${iPath}" http://${host}/files/issues) +if [ ! $? -eq 0 ] +then + exit $? +fi +case ${code} in + 201) + issueId=$(cat ${tmp}|jq .id) + echo "issues file: ${name} created. id=${issueId}" + ;; + *) + echo "create issues file - FAILED: ${code}." + cat ${tmp} + exit 1 +esac # -file=${aPath} -echo -n "--- +# Post deps. +code=$(curl -kSs -o ${tmp} -w "%{http_code}" -F "file=@${dPath}" http://${host}/files/deps) +if [ ! $? -eq 0 ] +then + exit $? +fi +case ${code} in + 201) + depId=$(cat ${tmp}|jq .id) + echo "deps file: ${name} created. id=${depId}" + ;; + *) + echo "create deps file - FAILED: ${code}." + cat ${tmp} + exit 1 +esac +# +# Post analysis. +d=" commit: "42b22a90" issues: + id: ${issueId} dependencies: -" > ${file} - -echo "Report CREATED" - -mime="application/x-yaml" + id: ${depId} +" +code=$(curl -kSs -o ${tmp} -w "%{http_code}" ${host}/applications/${appId}/analyses -H "Content-Type:application/x-yaml" -d "${d}") +if [ ! $? -eq 0 ] +then + exit $? +fi +case ${code} in + 201) + id=$(cat ${tmp}|jq .id) + echo "analysis: ${name} created. id=${id}" + ;; + *) + echo "create analysis - FAILED: ${code}." + cat ${tmp} + exit 1 +esac -curl \ - -F "file=@${aPath};type=${mime}" \ - -F "issues=@${iPath};type=${mime}" \ - -F "dependencies=@${dPath};type=${mime}" \ - ${host}/applications/${app}/analyses \ - -H "Accept:${mime}" +echo "Report POSTED" From 6a864543f42c883942f763cb62dc06cf84d25c8a Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Fri, 9 Aug 2024 09:02:43 -0700 Subject: [PATCH 2/2] checkpoint Signed-off-by: Jeff Ortel --- binding/application.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/binding/application.go b/binding/application.go index fab15516e..ac433c31c 100644 --- a/binding/application.go +++ b/binding/application.go @@ -333,16 +333,19 @@ func (h *Analysis) Create(commit, encoding, issues, deps string) (err error) { } r := api.AnalysisManifest{Commit: commit} file := File{client: h.client} + // post issues. f, err := file.Post(issues) if err != nil { return } r.Issues = api.Ref{ID: f.ID} + // post deps. f, err = file.Post(deps) if err != nil { return } r.Dependencies = api.Ref{ID: f.ID} + // post manifest. path := Path(api.AppAnalysesRoot).Inject(Params{api.ID: h.appId}) err = h.client.Encoding(encoding).Post(path, r) return