feat: compress coverage artifact before reporting (#204)

Makefile

@@ -5,9 +5,9 @@ build_local:
 	cd cmd/deepsource && go build -tags static_all -o /tmp/deepsource .
 
 test:
-	CGO_ENABLED=0 go test -v ./command/report/tests/... -count=1
+	CGO_ENABLED=1 go test -v ./command/report/tests/... -count=1
 	echo "\n====TESTING DEEPSOURCE PACKAGE====\n"
-	CGO_ENABLED=0 go test -v ./deepsource/tests/...
+	CGO_ENABLED=1 go test -v ./deepsource/tests/...
 	echo "\n====TESTING CONFIG VALIDATOR PACKAGE====\n"
 	go test -v ./configvalidator/... -count=1
 	echo "\n====CALCULATING TEST COVERAGE FOR ENTIRE PACKAGE====\n"

command/report/constants.go

@@ -29,6 +29,7 @@ Notes:
 Documentation:
     https://deepsource.io/docs/cli#report
 `
-	reportGraphqlQuery    = "mutation($input: CreateArtifactInput!) {\r\n  createArtifact(input: $input) {\r\n    ok\r\n  message\r\n  error\r\n  }\r\n}"
-	reportGraphqlQueryOld = "mutation($input: CreateArtifactInput!) {\r\n  createArtifact(input: $input) {\r\n    ok\r\n  error\r\n  }\r\n}"
+	reportGraphqlQuery     = "mutation($input: CreateArtifactInput!) {\r\n  createArtifact(input: $input) {\r\n    ok\r\n  message\r\n  error\r\n  }\r\n}"
+	reportGraphqlQueryOld  = "mutation($input: CreateArtifactInput!) {\r\n  createArtifact(input: $input) {\r\n    ok\r\n  error\r\n  }\r\n}"
+	graphqlCheckCompressed = "query {\r\n __type(name: \"ArtifactMetadataInput\") {\r\n inputFields {\r\n name\r\n }\r\n }\r\n}"
 )

command/report/report.go

@@ -1,13 +1,15 @@
 package report
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
 	"strings"
 	"time"
 
+	"github.com/DataDog/zstd"
 	"github.com/MakeNowJust/heredoc"
 	"github.com/deepsourcelabs/cli/utils"
 	"github.com/getsentry/sentry-go"
@@ -221,13 +223,74 @@ func (opts *ReportOptions) Run() int {
 		artifactValue = string(valueBytes)
 	}
 
+	// Query DeepSource API to check if compression is supported
+	q := ReportQuery{Query: graphqlCheckCompressed}
+
+	qBytes, err := json.Marshal(q)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "DeepSource | Error | Failed to marshal query:", err)
+		sentry.CaptureException(err)
+		return 1
+	}
+
+	r, err := makeQuery(
+		dsnProtocol+"://"+dsnHost+"/graphql/cli/",
+		qBytes,
+		"application/json",
+		opts.SkipCertificateVerification,
+	)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "DeepSource | Error | Failed to make query:", err)
+		sentry.CaptureException(err)
+		return 1
+	}
+
+	// res is a struct to unmarshal the response to check if compression is supported
+	var res struct {
+		Data struct {
+			Type struct {
+				InputFields []struct {
+					Name string `json:"name"`
+				} `json:"inputFields"`
+			} `json:"__type"`
+		} `json:"data"`
+	}
+
+	err = json.Unmarshal(r, &res)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "DeepSource | Error | Failed to unmarshal response:", err)
+		sentry.CaptureException(err)
+		return 1
+	}
+
+	reportMeta := make(map[string]interface{})
+	reportMeta["workDir"] = currentDir
+
+	// Compress the value if compression is supported
+	for _, inputField := range res.Data.Type.InputFields {
+		if inputField.Name == "compressed" {
+			// Compress the byte array
+			var compressedBytes []byte
+			compressLevel := 20
+			compressedBytes, err = zstd.CompressLevel(compressedBytes, []byte(artifactValue), compressLevel)
+			if err != nil {
+				fmt.Fprintln(os.Stderr, "DeepSource | Error | Failed to compress value file:", reportCommandValueFile)
+				sentry.CaptureException(err)
+				return 1
+			}
+
+			// Base64 encode the compressed byte array
+			artifactValue = base64.StdEncoding.EncodeToString(compressedBytes)
+
+			// Set the compression flag
+			reportMeta["compressed"] = "True"
+		}
+	}
+
 	////////////////////
 	// Generate query //
 	////////////////////
 
-	reportMeta := make(map[string]string)
-	reportMeta["workDir"] = currentDir
-
 	queryInput := ReportQueryInput{
 		AccessToken:       dsnAccessToken,
 		CommitOID:         headCommitOID,

command/report/tests/golden_files/report_grqphql_artifactmetadatainput_response_success.json

@@ -0,0 +1,14 @@
+{
+   "data": {
+      "__type": {
+         "inputFields": [
+            {
+               "name": "workDir"
+            },
+            {
+              "name": "compressed"
+            }
+         ]
+      }
+   }
+}

command/report/tests/report_workflow_test.go

@@ -2,13 +2,17 @@ package tests
 
 import (
 	"bytes"
+	"encoding/base64"
+	"encoding/json"
 	"io"
 	"log"
 	"net/http"
 	"os"
 	"os/exec"
 	"testing"
 
+	"github.com/DataDog/zstd"
+	"github.com/deepsourcelabs/cli/command/report"
 	"github.com/google/go-cmp/cmp"
 )
 
@@ -26,39 +30,97 @@ const (
 )
 
 func graphQLAPIMock(w http.ResponseWriter, r *http.Request) {
-	req, _ := io.ReadAll(r.Body)
-
-	// Read test graphql request body artifact file
-	requestBodyData, err := os.ReadFile("./golden_files/report_graphql_request_body.json")
+	// Read request request request body
+	req, err := io.ReadAll(r.Body)
 	if err != nil {
 		log.Println(err)
 		return
 	}
 
-	// Read test graphql success response body artifact file
-	successResponseBodyData, err := os.ReadFile("./golden_files/report_graphql_success_response_body.json")
-	if err != nil {
-		log.Println(err)
-		return
-	}
+	log.Println("Request body: ", string(req))
 
-	// Read test graphql error response body artifact file
-	errorResponseBodyData, err := os.ReadFile("./golden_files/report_graphql_error_response_body.json")
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	w.WriteHeader(http.StatusOK)
-	w.Header().Set("Content-Type", "application/json")
+	// Check if the request has ArtifactMetadataInput in body
+	if bytes.Contains(req, []byte("ArtifactMetadataInput")) {
+		log.Println("ArtifactMetadataInput found in request body")
+		w.WriteHeader(http.StatusOK)
+		w.Header().Set("Content-Type", "application/json")
 
-	if want, got := string(requestBodyData), string(req); want == got {
+		successResponseBodyData, err := os.ReadFile("./golden_files/report_grqphql_artifactmetadatainput_response_success.json")
+		if err != nil {
+			log.Println(err)
+			return
+		}
 		w.Write([]byte(successResponseBodyData))
+
 	} else {
-		if want != got {
-			log.Printf("Mismatch found:\nDiff: %s\n", cmp.Diff(want, got))
+
+		// Unmarshal request body into ReportQuery
+		var reportQuery report.ReportQuery
+		err = json.Unmarshal(req, &reportQuery)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+
+		requestData := reportQuery.Variables.Input.Data
+
+		// Decode base64 encoded data
+		decodedData, err := base64.StdEncoding.DecodeString(requestData)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+
+		// Decompress zstd compressed data
+		decompressedData, err := zstd.Decompress(nil, decodedData)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+
+		// Create new ReportQeury object with decompressed data
+		reportQuery.Variables.Input.Data = string(decompressedData)
+
+		// Read test graphql request body artifact file
+		requestBodyData, err := os.ReadFile("./golden_files/report_graphql_request_body.json")
+		if err != nil {
+			log.Println(err)
+			return
+		}
+
+		// Unmarshal request body into ReportQuery
+		var requestReportQuery report.ReportQuery
+		err = json.Unmarshal(requestBodyData, &requestReportQuery)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+
+		// Read test graphql success response body artifact file
+		successResponseBodyData, err := os.ReadFile("./golden_files/report_graphql_success_response_body.json")
+		if err != nil {
+			log.Println(err)
+			return
+		}
+
+		// Read test graphql error response body artifact file
+		errorResponseBodyData, err := os.ReadFile("./golden_files/report_graphql_error_response_body.json")
+		if err != nil {
+			log.Println(err)
+			return
+		}
+
+		w.WriteHeader(http.StatusOK)
+		w.Header().Set("Content-Type", "application/json")
+
+		if want, got := requestReportQuery, reportQuery; cmp.Equal(want, got) {
+			w.Write([]byte(successResponseBodyData))
+		} else {
+			if want != got {
+				log.Printf("Mismatch found:\nDiff: %s\n", cmp.Diff(want, got))
+			}
+			w.Write([]byte(errorResponseBodyData))
 		}
-		w.Write([]byte(errorResponseBodyData))
 	}
 }
 

go.mod

@@ -4,6 +4,7 @@ go 1.18
 
 require (
 	github.com/AlecAivazis/survey/v2 v2.2.12
+	github.com/DataDog/zstd v1.5.5
 	github.com/Jeffail/gabs/v2 v2.6.1
 	github.com/MakeNowJust/heredoc v1.0.0
 	github.com/cli/browser v1.1.0

go.sum

@@ -18,6 +18,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw=
 github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w=
+github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ=
+github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/Jeffail/gabs/v2 v2.6.1 h1:wwbE6nTQTwIMsMxzi6XFQQYRZ6wDc1mSdxoAN+9U4Gk=
 github.com/Jeffail/gabs/v2 v2.6.1/go.mod h1:xCn81vdHKxFUuWWAaD5jCTQDNPBMh5pPs9IJ+NcziBI=
 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=