Syncing beta with latest stage (#94)

* Init git while cloning (#67) * Init git while cloning * fixed tests * Fixed logging * synapse ws reconnect (#42) * added retry websocket connection logic * added connection abort functionality if duplicate connection requested * added entry in go.mod and go.sum , change restart policy in docker-compose * fix logout when graceful shutdown * refactor code and resolve review comments * change gracefultimeout for synapse from ms to s * update json keys for test suite payload (#70) * update json keys for test suite payload * fixed unit test cases * added refresh token api (#69) * added refresh token api * added unit tests * added unit test for secret parser expired function * disable golangci for false positive case * added comments * Update README.md (#58) (#76) Co-authored-by: Saurabh Prakash <saurabhp@lambdatest.com> * updated readme for ambiguity (#77) updated readme for ambiguity * Update main.yml (#79) * Feature/synapse bitbucket (#80) * added refresh token api * added unit tests * added unit test for secret parser expired function * disable golangci for false positive case * added comments * added token type * added gitprovider condition in refresh oauth * added basic auth supprt in git init * fixed spelling * fixed unit tests * fixed unit test * Changes get oauth secret test * Bugfix/bitbucket fork pr (#84) * added refresh token api * added unit tests * added unit test for secret parser expired function * disable golangci for false positive case * added comments * added token type * added gitprovider condition in refresh oauth * added basic auth supprt in git init * fixed spelling * fixed unit tests * fixed unit test * Changes get oauth secret test * updated readme for ambiguity (#77) updated readme for ambiguity * Update main.yml (#79) * added bitbucket forked pr support Co-authored-by: Nevil Macwan <93511721+nevilm-lt@users.noreply.github.com> Co-authored-by: Saurabh Prakash <saurabhp@lambdatest.com> * Removed parsing pod (#52) * Removed parsing pod * stable * updated tests * Added golangci lint file * Disabled lint checks for unblocking this change * Addressed PR comments Co-authored-by: VikrantKS <96419531+VikrantKS@users.noreply.github.com> Co-authored-by: palashmarantas <91466368+palashmarantas@users.noreply.github.com> Co-authored-by: utkarsh-lambdatest <98745310+utkarsh-lambdatest@users.noreply.github.com> Co-authored-by: Nevil Macwan <93511721+nevilm-lt@users.noreply.github.com>
LambdaTest · Mar 28, 2022 · 4725803 · 4725803
1 parent d4bf3cd
commit 4725803
Show file tree

Hide file tree

Showing 54 changed files with 1,026 additions and 672 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -80,7 +80,6 @@ jobs:
           issues_threshold=13
           gofmt_score_threshold=100
           go_vet_score_threshold=100
-          golint_score_threshold=98
           gocyclo_score_threshold=91
           git clone https://github.com/gojp/goreportcard.git
           cd goreportcard
@@ -93,7 +92,6 @@ jobs:
           issues=$(cat reportcard.txt| grep 'Issues:' | awk  '{print $2}' | tr -d \%)
           gofmt_score=$(cat reportcard.txt| grep 'gofmt:' | awk  '{print $2}' | tr -d \%)
           go_vet_score=$(cat reportcard.txt| grep 'go_vet:' | awk  '{print $2}' | tr -d \%)
-          golint_score=$(cat reportcard.txt| grep 'golint:' | awk  '{print $2}' | tr -d \%)
           gocyclo_score=$(cat reportcard.txt| grep 'gocyclo:' | awk  '{print $2}' | tr -d \%)
           rm reportcard.txt
           failed_checks=0
@@ -110,10 +108,6 @@ jobs:
             failure_reason="${failure_reason}\ngo-vet score: $go_vet_score. Threshold was: $go_vet_score_threshold."
             ((failed_checks+=1))
           fi
-          if [[ $golint_score -lt $golint_score_threshold ]]; then
-            failure_reason="${failure_reason}\ngo-lint score: $golint_score. Threshold was: $golint_score_threshold."
-            ((failed_checks+=1))
-          fi
           if [[ $gocyclo_score -lt $gocyclo_score_threshold ]]; then
             failure_reason="${failure_reason}\ngo-cyclo score: $gocyclo_score. Threshold was: $gocyclo_score_threshold."
             ((failed_checks+=1))

diff --git a/.golangci.yml b/.golangci.yml
@@ -0,0 +1,136 @@
+# # Uncomment following lines after fixing linting issues in test files
+#
+# linters-settings:
+#   depguard:
+#     list-type: blacklist
+#   funlen:
+#     lines: 100
+#     statements: 50
+#   gci:
+#     local-prefixes: github.com/golangci/golangci-lint
+#   goconst:
+#     min-len: 2
+#     min-occurrences: 2
+#   gocritic:
+#     enabled-tags:
+#       - diagnostic
+#       - experimental
+#       - opinionated
+#       - performance
+#       - style
+#     disabled-checks:
+#       - dupImport # https://github.com/go-critic/go-critic/issues/845
+#       - ifElseChain
+#       - octalLiteral
+#       - whyNoLint
+#       - wrapperFunc
+#   gocyclo:
+#     min-complexity: 15
+#   goimports:
+#     local-prefixes: github.com/golangci/golangci-lint
+#   gomnd:
+#     settings:
+#       mnd:
+#         # don't include the "operation" and "assign"
+#         checks:
+#           - argument
+#           - case
+#           - condition
+#           - return
+#   govet:
+#     check-shadowing: true
+#     settings:
+#       printf:
+#         funcs:
+#           - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
+#           - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
+#           - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
+#           - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
+#   lll:
+#     line-length: 140
+#   maligned:
+#     suggest-new: true
+#   misspell:
+#     locale: US
+#   nolintlint:
+#     allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
+#     allow-unused: false # report any unused nolint directives
+#     require-explanation: false # don't require an explanation for nolint directives
+#     require-specific: false # don't require nolint directives to be specific about which linter is being skipped
+
+# linters:
+#   disable-all: true
+#   enable:
+#     - bodyclose
+#     - deadcode
+#     - depguard
+#     - dogsled
+#     - dupl
+#     - errcheck
+#     - exportloopref
+#     - exhaustive
+#     - funlen
+#     - gochecknoinits
+#     - goconst
+#     - gocritic
+#     - gocyclo
+#     - gofmt
+#     - goimports
+#     - gomnd
+#     - goprintffuncname
+#     - gosec
+#     - gosimple
+#     - govet
+#     - ineffassign
+#     - lll
+#     - misspell
+#     - nakedret
+#     - noctx
+#     - nolintlint
+#     - rowserrcheck
+#     - staticcheck
+#     - structcheck
+#     - stylecheck
+#     - typecheck
+#     - unconvert
+#     - unparam
+#     - unused
+#     - varcheck
+#     - whitespace
+
+#   # don't enable:
+#   # - asciicheck
+#   # - scopelint
+#   # - gochecknoglobals
+#   # - gocognit
+#   # - godot
+#   # - godox
+#   # - goerr113
+#   # - interfacer
+#   # - maligned
+#   # - nestif
+#   # - prealloc
+#   # - testpackage
+#   # - revive
+#   # - wsl
+
+# issues:
+#   # Excluding configuration per-path, per-linter, per-text and per-source
+#   exclude-rules:
+#     - path: _test\.go
+#       linters:
+#         - gomnd
+
+#     - path: pkg/golinters/errcheck.go
+#       text: "SA1019: errCfg.Exclude is deprecated: use ExcludeFunctions instead"
+#     - path: pkg/commands/run.go
+#       text: "SA1019: lsc.Errcheck.Exclude is deprecated: use ExcludeFunctions instead"
+
+# run:
+#   timeout: 2m
+#   skip-dirs:
+#     - test/testdata_etc
+#     - internal/cache
+#     - internal/renameio
+#     - internal/robustio
+#     - pkg/docs
diff --git a/.sample.synapse.json b/.sample.synapse.json
@@ -7,7 +7,8 @@
       "SecretKey": "SecretKeyForLambdaTest"
     },
     "Git": {
-      "Token": ""
+      "Token": "",
+      "TokenType": "Bearer"
     },
     "ContainerRegistry": {
       "PullPolicy": "always",

diff --git a/README.md b/README.md
@@ -10,8 +10,8 @@
 </p>
 
 <p align="center">
-  <a href="https://github.com/LambdaTest/test-at-scale/blob/master/LICENSE"><img src="https://img.shields.io/badge/license-Apache%20License%202.0.-blue" /></a>
-  <a href="https://github.com/LambdaTest/test-at-scale/blob/master/CONTRIBUTING.md"><img src="https://img.shields.io/badge/contributions-welcome-brightgreen?logo=github" /></a>
+  <a href="https://github.com/LambdaTest/test-at-scale/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%20License%202.0.-blue" /></a>
+  <a href="https://github.com/LambdaTest/test-at-scale/blob/main/CONTRIBUTING.md"><img src="https://img.shields.io/badge/contributions-welcome-brightgreen?logo=github" /></a>
   <a href="#build"><img src="https://github.com/lambdatest/test-at-scale/actions/workflows/main.yml/badge.svg" /></a>
   <a href="#lint"><img src="https://github.com/lambdatest/test-at-scale/actions/workflows/golangci-lint.yml/badge.svg" /></a>
   <a href="#stale"><img src="https://github.com/lambdatest/test-at-scale/actions/workflows/stale.yml/badge.svg" /></a>
@@ -76,7 +76,7 @@ Before installation we need to create a file that will be used for configuring t
 ```bash
 mkdir ~/test-at-scale
 cd ~/test-at-scale
-curl https://raw.githubusercontent.com/LambdaTest/test-at-scale/master/.sample.synapse.json -o .synapse.json
+curl https://raw.githubusercontent.com/LambdaTest/test-at-scale/main/.sample.synapse.json -o .synapse.json
 ```
 - Open the downloaded `.synapse.json` configuration file in any editor of your choice.
 - You will need to add the following in this file: 
@@ -91,7 +91,7 @@ curl https://raw.githubusercontent.com/LambdaTest/test-at-scale/master/.sample.s
 #### Installation on Docker
 
 ##### Prerequisites
-- [Docker](https://docs.docker.com/get-docker/) or [Docker-Compose](https://docs.docker.com/compose/install/) (Recommended)
+- [Docker](https://docs.docker.com/get-docker/) and [Docker-Compose](https://docs.docker.com/compose/install/) (Recommended)
 
 ##### Docker Compose
 - Run the docker application.
@@ -207,4 +207,4 @@ We are committed to fostering an open and welcoming environment in the community
 
 ## **License**
 
-TestAtScale is available under the [Apache License 2.0](https://github.com/LambdaTest/test-at-scale/blob/master/LICENSE). Use it wisely.
+TestAtScale is available under the [Apache License 2.0](https://github.com/LambdaTest/test-at-scale/blob/main/LICENSE). Use it wisely.
diff --git a/cmd/nucleus/bin.go b/cmd/nucleus/bin.go
@@ -25,10 +25,10 @@ import (
 	"github.com/LambdaTest/synapse/pkg/global"
 	"github.com/LambdaTest/synapse/pkg/lumber"
 	"github.com/LambdaTest/synapse/pkg/payloadmanager"
+	"github.com/LambdaTest/synapse/pkg/requestutils"
 	"github.com/LambdaTest/synapse/pkg/secret"
 	"github.com/LambdaTest/synapse/pkg/server"
 	"github.com/LambdaTest/synapse/pkg/service/coverage"
-	"github.com/LambdaTest/synapse/pkg/service/parser"
 	"github.com/LambdaTest/synapse/pkg/service/teststats"
 	"github.com/LambdaTest/synapse/pkg/tasconfigmanager"
 	"github.com/LambdaTest/synapse/pkg/task"
@@ -114,18 +114,21 @@ func run(cmd *cobra.Command, args []string) {
 	pm := payloadmanager.NewPayloadManger(azureClient, logger, cfg)
 	secretParser := secret.New(logger)
 	tcm := tasconfigmanager.NewTASConfigManager(logger)
-	gm := gitmanager.NewGitManager(logger)
-	dm := diffmanager.NewDiffManager(cfg, logger)
+	requests := requestutils.New(logger)
 	execManager := command.NewExecutionManager(secretParser, azureClient, logger)
-	tds := testdiscoveryservice.NewTestDiscoveryService(execManager, logger)
+	gm := gitmanager.NewGitManager(logger, execManager)
+	dm := diffmanager.NewDiffManager(cfg, logger)
+
+	tdResChan := make(chan core.DiscoveryResult)
+	tds := testdiscoveryservice.NewTestDiscoveryService(ctx, tdResChan, execManager, requests, logger)
 	tes := testexecutionservice.NewTestExecutionService(execManager, azureClient, ts, logger)
 	tbs, err := blocktestservice.NewTestBlockTestService(cfg, logger)
 	if err != nil {
 		logger.Fatalf("failed to initialize test blocklist service: %v", err)
 	}
-	router := api.NewRouter(logger, ts)
+	router := api.NewRouter(logger, ts, tdResChan)
 
-	t, err := task.New(ctx, cfg, logger)
+	t, err := task.New(ctx, requests, logger)
 	if err != nil {
 		logger.Fatalf("failed to initialize task: %v", err)
 	}
@@ -139,8 +142,6 @@ func run(cmd *cobra.Command, args []string) {
 		logger.Fatalf("failed to initialize cache manager: %v", err)
 	}
 
-	parserService := parser.New(tcm, logger)
-
 	coverageService, err := coverage.New(execManager, azureClient, zstd, cfg, logger)
 	if err != nil {
 		logger.Fatalf("failed to initialize coverage service: %v", err)
@@ -154,7 +155,6 @@ func run(cmd *cobra.Command, args []string) {
 	pl.BlockTestService = tbs
 	pl.TestExecutionService = tes
 	pl.ExecutionManager = execManager
-	pl.ParserService = parserService
 	pl.CoverageService = coverageService
 	pl.TestStats = ts
 	pl.Task = t

diff --git a/cmd/nucleus/flags.go b/cmd/nucleus/flags.go
@@ -12,7 +12,6 @@ func AttachCLIFlags(rootCmd *cobra.Command) error {
 	rootCmd.PersistentFlags().StringP("payloadAddress", "l", "", "Payload address")
 	rootCmd.PersistentFlags().BoolP("verbose", "", false, "Run in verbose mode")
 	rootCmd.PersistentFlags().BoolP("coverage", "", false, "Run coverage only mode")
-	rootCmd.PersistentFlags().BoolP("parser", "", false, "Run YML parsing only mode")
 	rootCmd.PersistentFlags().BoolP("discover", "", false, "Run nucleus in test discovery mode")
 	rootCmd.PersistentFlags().BoolP("execute", "", false, "Run nucleus in test execution mode")
 	rootCmd.PersistentFlags().StringP("env", "e", "prod", "Environment.")

diff --git a/cmd/synapse/bin.go b/cmd/synapse/bin.go
@@ -95,8 +95,11 @@ func run(cmd *cobra.Command, args []string) {
 		logger.Fatalf("Could not instantiate proxyhandler %v", err)
 	}
 
+	// All attempts to connect to lambdatest server failed
+	connectionFailed := make(chan struct{})
+
 	wg.Add(1)
-	go synapse.InitiateConnection(ctx, &wg)
+	go synapse.InitiateConnection(ctx, &wg, connectionFailed)
 
 	wg.Add(1)
 	go func() {
@@ -138,6 +141,21 @@ func run(cmd *cobra.Command, args []string) {
 			case <-time.After(global.GracefulTimeout):
 				logger.Errorf("Graceful timeout exceeded. Brutally killing the application")
 			}
+		}
+
+	case <-connectionFailed:
+		{
+			logger.Debugf("main: all attempts to connect to lamdatest server failed ....")
+			// tell the goroutines to stop
+			logger.Debugf("main: telling goroutines to stop")
+			cancel()
+			select {
+			case <-done:
+				logger.Debugf("Go routines exited within timeout")
+			case <-time.After(global.GracefulTimeout):
+				logger.Errorf("Graceful timeout exceeded. Brutally killing the application")
+			}
+			os.Exit(0)
 
 		}
 	case <-done:

diff --git a/config/nucleusmodel.go b/config/nucleusmodel.go
@@ -12,7 +12,6 @@ type NucleusConfig struct {
 	LogFile        string
 	LogConfig      lumber.LoggingConfig
 	CoverageMode   bool   `json:"coverage" yaml:"coverageOnly"`
-	ParseMode      bool   `json:"parser" yaml:"parseOnly"`
 	DiscoverMode   bool   `json:"discover" yaml:"discoverOnly"`
 	ExecuteMode    bool   `json:"execute" yaml:"executeOnly"`
 	TaskID         string `json:"taskID" env:"TASK_ID"`

diff --git a/config/synapsemodel.go b/config/synapsemodel.go
@@ -24,7 +24,8 @@ type LambdatestConfig struct {
 
 // GitConfig contains git token
 type GitConfig struct {
-	Token string
+	Token     string
+	TokenType string
 }
 
 // PullPolicyType defines when to pull docker image

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -3,7 +3,7 @@ services:
   synapse:
     image: lambdatest/synapse:latest
     stop_signal: SIGINT
-    restart: always
+    restart: on-failure
     networks:
       - test-at-scale
     hostname: synapse

diff --git a/go.mod b/go.mod
@@ -15,6 +15,7 @@ require (
 	github.com/google/uuid v1.2.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/joho/godotenv v1.4.0
+	github.com/lestrrat-go/backoff v1.0.1
 	github.com/mholt/archiver/v3 v3.5.1
 	github.com/shirou/gopsutil/v3 v3.21.1
 	github.com/sirupsen/logrus v1.8.1

diff --git a/go.sum b/go.sum
@@ -579,6 +579,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
 github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
+github.com/lestrrat-go/backoff v1.0.1 h1:Gphaach0QvvtaHmR9U8hwXNHXWckPyD8V6S+V+D184c=
+github.com/lestrrat-go/backoff v1.0.1/go.mod h1:5QVJKC49Q5yQvCrpup0ZqzDGHEO/O4H82cnDuYdumkw=
 github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=

diff --git a/pkg/api/router.go b/pkg/api/router.go
@@ -3,6 +3,8 @@ package api
 import (
 	"github.com/LambdaTest/synapse/pkg/api/health"
 	"github.com/LambdaTest/synapse/pkg/api/results"
+	"github.com/LambdaTest/synapse/pkg/api/testlist"
+	"github.com/LambdaTest/synapse/pkg/core"
 	"github.com/LambdaTest/synapse/pkg/lumber"
 	"github.com/LambdaTest/synapse/pkg/service/teststats"
 	"github.com/gin-gonic/gin"
@@ -12,13 +14,15 @@ import (
 type Router struct {
 	logger           lumber.Logger
 	testStatsService *teststats.ProcStats
+	tdResChan        chan core.DiscoveryResult
 }
 
 // NewRouter returns instance of Router
-func NewRouter(logger lumber.Logger, ts *teststats.ProcStats) Router {
+func NewRouter(logger lumber.Logger, ts *teststats.ProcStats, tdResChan chan core.DiscoveryResult) Router {
 	return Router{
 		logger:           logger,
 		testStatsService: ts,
+		tdResChan:        tdResChan,
 	}
 }
 
@@ -33,6 +37,7 @@ func (r Router) Handler() *gin.Engine {
 	// router.Use(cors.New(corsConfig))
 	router.GET("/health", health.Handler)
 	router.POST("/results", results.Handler(r.logger, r.testStatsService))
+	router.POST("/test-list", testlist.Handler(r.logger, r.tdResChan))
 
 	return router