Syncing main with latest beta

.github/workflows/main.yml

@@ -4,11 +4,20 @@ on:
   push:
     branches:
       - main
+      - beta
+      - dev
+    tags: [ 'v*.*.*' ]
 
-
   pull_request:
     branches:
       - main
+      - beta
+      - dev
+
+  workflow_dispatch:
+
+env:
+  REGISTRY_NAME: lambdatest
 
 jobs:
   Unit_Test_Cases:
@@ -55,6 +64,7 @@ jobs:
         run: |
           go test -parallel 4 -coverpkg=./... -coverprofile=profile.cov ./...
           go tool cover -func profile.cov
+
   Go_Report_Card:
     runs-on: ubuntu-latest
     needs: [ Unit_Test_Cases, Performance_Test_Cases ]
@@ -109,3 +119,55 @@ jobs:
           else
             exit 0
           fi
+
+  Build_To_Deploy:
+    if: ${{ github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || github.ref == 'refs/heads/main' }}
+    runs-on: ubuntu-latest
+    needs: [ Test_Coverage, Go_Report_Card ]
+    steps:
+      - name: Inject slug/short variables
+        uses: rlespinasse/github-slug-action@v3.x
+
+      - name: Checkout
+        uses: actions/checkout@v2.4.0
+
+      - name: Build Cloud Runners
+        if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/beta' || github.ref == 'refs/heads/dev' }}
+        run: |
+          curl -X  POST "https://api.github.com/repos/LambdaTest/test-at-scale-deployment/actions/workflows/main.yml/dispatches" -H "Authorization:Bearer ${{ secrets.GH_API_TOKEN }}" -d '{"ref":"${{env.GITHUB_REF_SLUG}}"}'
+
+      - name: Setup Docker Image Tags
+        run: |
+          if [ ${GITHUB_REF_SLUG} == "main" ] ; then
+            echo "NUCLEUS_TAGS=${REGISTRY_NAME}/nucleus:base" >> $GITHUB_ENV
+            echo "SYNAPSE_TAGS=${REGISTRY_NAME}/synapse:latest","${REGISTRY_NAME}/synapse:prod" >> $GITHUB_ENV
+          else
+            echo "NUCLEUS_TAGS=${REGISTRY_NAME}/nucleus:base-${GITHUB_REF_SLUG}" >> $GITHUB_ENV
+            echo "SYNAPSE_TAGS=${REGISTRY_NAME}/synapse:${GITHUB_REF_SLUG}" >> $GITHUB_ENV
+          fi
+
+      - name: Docker Login
+        uses: docker/login-action@v1.13.0
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+          logout: true
+
+      - name: Build and push Nucleus images
+        if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/beta' || github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/v') }}
+        uses: docker/build-push-action@v2.9.0
+        with:
+          context: .
+          tags: ${{ env.NUCLEUS_TAGS }}
+          file: build/nucleus/Dockerfile
+          push: true
+
+      - name: Build and push Synapse images
+        if: ${{ github.ref == 'refs/heads/beta' || github.ref == 'refs/heads/dev' ||  github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }}
+        uses: docker/build-push-action@v2.9.0
+        with:
+          context: .
+          tags: ${{ env.SYNAPSE_TAGS }}
+          file: build/synapse/Dockerfile
+          push: true
+

.gitignore

@@ -79,6 +79,5 @@ mould.log
 
 .env
 
-testdata
-
 .synapse.json
+logs

.golangci.yml

@@ -0,0 +1,136 @@
+# Uncomment following lines after fixing linting issues in test files
+
+linters-settings:
+  depguard:
+    list-type: blacklist
+  funlen:
+    lines: 100
+    statements: 50
+  gci:
+    local-prefixes: github.com/golangci/golangci-lint
+  goconst:
+    min-len: 2
+    min-occurrences: 2
+  gocritic:
+    enabled-tags:
+      - diagnostic
+      - experimental
+      - opinionated
+      - performance
+      - style
+    disabled-checks:
+      - dupImport # https://github.com/go-critic/go-critic/issues/845
+      - ifElseChain
+      - octalLiteral
+      - whyNoLint
+      - wrapperFunc
+  gocyclo:
+    min-complexity: 15
+  goimports:
+    local-prefixes: github.com/golangci/golangci-lint
+  gomnd:
+    settings:
+      mnd:
+        # don't include the "operation" and "assign"
+        checks:
+          - argument
+          - case
+          - condition
+          - return
+  govet:
+    check-shadowing: true
+    settings:
+      printf:
+        funcs:
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
+  lll:
+    line-length: 140
+  maligned:
+    suggest-new: true
+  misspell:
+    locale: US
+  nolintlint:
+    allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
+    allow-unused: false # report any unused nolint directives
+    require-explanation: false # don't require an explanation for nolint directives
+    require-specific: false # don't require nolint directives to be specific about which linter is being skipped
+
+linters:
+  disable-all: true
+  enable:
+    - bodyclose
+    - deadcode
+    - depguard
+    - dogsled
+    - dupl
+    - errcheck
+    - exportloopref
+    - exhaustive
+    - funlen
+    - gochecknoinits
+    - goconst
+    - gocritic
+    - gocyclo
+    - gofmt
+    - goimports
+    - gomnd
+    - goprintffuncname
+    - gosec
+    - gosimple
+    - govet
+    - ineffassign
+    - lll
+    - misspell
+    - nakedret
+    - noctx
+    - nolintlint
+    - rowserrcheck
+    - staticcheck
+    - structcheck
+    - stylecheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+    - varcheck
+    - whitespace
+
+  # don't enable:
+  # - asciicheck
+  # - scopelint
+  # - gochecknoglobals
+  # - gocognit
+  # - godot
+  # - godox
+  # - goerr113
+  # - interfacer
+  # - maligned
+  # - nestif
+  # - prealloc
+  # - testpackage
+  # - revive
+  # - wsl
+
+issues:
+  # Excluding configuration per-path, per-linter, per-text and per-source
+  exclude-rules:
+    - path: _test\.go
+      linters:
+        - gomnd
+
+    - path: pkg/golinters/errcheck.go
+      text: "SA1019: errCfg.Exclude is deprecated: use ExcludeFunctions instead"
+    - path: pkg/commands/run.go
+      text: "SA1019: lsc.Errcheck.Exclude is deprecated: use ExcludeFunctions instead"
+
+run:
+  timeout: 2m
+  skip-dirs:
+    - test/testdata_etc
+    - internal/cache
+    - internal/renameio
+    - internal/robustio
+    - pkg/docs

.sample.synapse.json

@@ -1,14 +1,14 @@
 {
     "logConfig": {
       "enableConsole": true,
-      "consoleJSONFormat": true,
       "consolelevel": "debug"
     },
     "Lambdatest": {
       "SecretKey": "SecretKeyForLambdaTest"
     },
     "Git": {
-      "Token": ""
+      "Token": "",
+      "TokenType": "Bearer"
     },
     "ContainerRegistry": {
       "PullPolicy": "always",

Makefile

@@ -7,14 +7,17 @@ SYNAPSE_IMAGE_NAME ?= lambdatest/synapse:latest
 usage:						## Show this help
 	@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/:.*##\s*/##/g' | awk -F'##' '{ printf "%-25s -> %s\n", $$1, $$2 }'
 
-build-nucleus-image:		## builds nucleus docker image
+lint:						## Runs linting
+	golangci-lint run
+
+build-nucleus-image:		## Builds nucleus docker image
 	docker build -t ${NUCLEUS_IMAGE_NAME} --file $(NUCLEUS_DOCKER_FILE) .
 
-build-nucleus-bin:			## builds nucleus binary
+build-nucleus-bin:			## Builds nucleus binary
 	bash build/nucleus/build.sh
 
-build-synapse-image:		## builds synapse docker image
+build-synapse-image:		## Builds synapse docker image
 	docker build -t ${SYNAPSE_IMAGE_NAME} --file $(SYNAPSE_DOCKER_FILE) .
 
-build-synapse-bin:			## builds synapse binary
+build-synapse-bin:			## Builds synapse binary
 	bash build/synapse/build.sh

README.md

@@ -93,7 +93,7 @@ curl https://raw.githubusercontent.com/LambdaTest/test-at-scale/main/.sample.syn
 #### Installation on Docker
 
 ##### Prerequisites
-- [Docker](https://docs.docker.com/get-docker/) or [Docker-Compose](https://docs.docker.com/compose/install/) (Recommended)
+- [Docker](https://docs.docker.com/get-docker/) and [Docker-Compose](https://docs.docker.com/compose/install/) (Recommended)
 
 ##### Docker Compose
 - Run the docker application.

build/nucleus/Dockerfile

@@ -28,9 +28,9 @@ WORKDIR /tmp/custom-runners
 RUN npm init -y
 
 RUN npm i --global-style --legacy-peer-deps \
-    @lambdatest/test-at-scale-jasmine-runner@~0.0.1 \
-    @lambdatest/test-at-scale-mocha-runner@~0.0.1 \
-    @lambdatest/test-at-scale-jest-runner@~0.0.1
+    @lambdatest/test-at-scale-jasmine-runner@~0.1.0 \
+    @lambdatest/test-at-scale-mocha-runner@~0.1.0 \
+    @lambdatest/test-at-scale-jest-runner@~0.1.0
 RUN npm i -g nyc@^15.1.0
 
 RUN tar -zcf /custom-runners/custom-runners.tgz node_modules

cmd/nucleus/bin.go

@@ -16,6 +16,7 @@ import (
 	"github.com/LambdaTest/synapse/config"
 	"github.com/LambdaTest/synapse/pkg/api"
 	"github.com/LambdaTest/synapse/pkg/azure"
+	"github.com/LambdaTest/synapse/pkg/blocktestservice"
 	"github.com/LambdaTest/synapse/pkg/cachemanager"
 	"github.com/LambdaTest/synapse/pkg/command"
 	"github.com/LambdaTest/synapse/pkg/core"
@@ -24,14 +25,13 @@ import (
 	"github.com/LambdaTest/synapse/pkg/global"
 	"github.com/LambdaTest/synapse/pkg/lumber"
 	"github.com/LambdaTest/synapse/pkg/payloadmanager"
+	"github.com/LambdaTest/synapse/pkg/requestutils"
 	"github.com/LambdaTest/synapse/pkg/secret"
 	"github.com/LambdaTest/synapse/pkg/server"
 	"github.com/LambdaTest/synapse/pkg/service/coverage"
-	"github.com/LambdaTest/synapse/pkg/service/parser"
 	"github.com/LambdaTest/synapse/pkg/service/teststats"
 	"github.com/LambdaTest/synapse/pkg/tasconfigmanager"
 	"github.com/LambdaTest/synapse/pkg/task"
-	"github.com/LambdaTest/synapse/pkg/testblocklistservice"
 	"github.com/LambdaTest/synapse/pkg/testdiscoveryservice"
 	"github.com/LambdaTest/synapse/pkg/testexecutionservice"
 	"github.com/LambdaTest/synapse/pkg/zstd"
@@ -114,18 +114,21 @@ func run(cmd *cobra.Command, args []string) {
 	pm := payloadmanager.NewPayloadManger(azureClient, logger, cfg)
 	secretParser := secret.New(logger)
 	tcm := tasconfigmanager.NewTASConfigManager(logger)
-	gm := gitmanager.NewGitManager(logger)
-	dm := diffmanager.NewDiffManager(cfg, logger)
+	requests := requestutils.New(logger)
 	execManager := command.NewExecutionManager(secretParser, azureClient, logger)
-	tds := testdiscoveryservice.NewTestDiscoveryService(execManager, logger)
+	gm := gitmanager.NewGitManager(logger, execManager)
+	dm := diffmanager.NewDiffManager(cfg, logger)
+
+	tdResChan := make(chan core.DiscoveryResult)
+	tds := testdiscoveryservice.NewTestDiscoveryService(ctx, tdResChan, execManager, requests, logger)
 	tes := testexecutionservice.NewTestExecutionService(execManager, azureClient, ts, logger)
-	tbs, err := testblocklistservice.NewTestBlockListService(cfg, logger)
+	tbs, err := blocktestservice.NewTestBlockTestService(cfg, logger)
 	if err != nil {
 		logger.Fatalf("failed to initialize test blocklist service: %v", err)
 	}
-	router := api.NewRouter(logger, ts)
+	router := api.NewRouter(logger, ts, tdResChan)
 
-	t, err := task.New(ctx, cfg, logger)
+	t, err := task.New(ctx, requests, logger)
 	if err != nil {
 		logger.Fatalf("failed to initialize task: %v", err)
 	}
@@ -139,8 +142,6 @@ func run(cmd *cobra.Command, args []string) {
 		logger.Fatalf("failed to initialize cache manager: %v", err)
 	}
 
-	parserService := parser.New(tcm, logger)
-
 	coverageService, err := coverage.New(execManager, azureClient, zstd, cfg, logger)
 	if err != nil {
 		logger.Fatalf("failed to initialize coverage service: %v", err)
@@ -151,10 +152,9 @@ func run(cmd *cobra.Command, args []string) {
 	pl.GitManager = gm
 	pl.DiffManager = dm
 	pl.TestDiscoveryService = tds
-	pl.TestBlockListService = tbs
+	pl.BlockTestService = tbs
 	pl.TestExecutionService = tes
 	pl.ExecutionManager = execManager
-	pl.ParserService = parserService
 	pl.CoverageService = coverageService
 	pl.TestStats = ts
 	pl.Task = t

cmd/nucleus/flags.go

@@ -12,7 +12,6 @@ func AttachCLIFlags(rootCmd *cobra.Command) error {
 	rootCmd.PersistentFlags().StringP("payloadAddress", "l", "", "Payload address")
 	rootCmd.PersistentFlags().BoolP("verbose", "", false, "Run in verbose mode")
 	rootCmd.PersistentFlags().BoolP("coverage", "", false, "Run coverage only mode")
-	rootCmd.PersistentFlags().BoolP("parser", "", false, "Run YML parsing only mode")
 	rootCmd.PersistentFlags().BoolP("discover", "", false, "Run nucleus in test discovery mode")
 	rootCmd.PersistentFlags().BoolP("execute", "", false, "Run nucleus in test execution mode")
 	rootCmd.PersistentFlags().StringP("env", "e", "prod", "Environment.")