[AI-1561] Split server and client in different packages. (#28)

* Fixed local environment. Added container to run the API locally and run tests. Fixed makefile

* Decouple Server/Client tests

* Bootstrap all Server folder. Moved CMD, main and common packages

* Moved all server files and test to server/ folder

* Moved test getConfig to test suite file

* Fixed client side tests

* Removed unused client cmd

* Removed all server packages from client package

* Moved client load-test and producer modules to testing module

* Fix cross ref import

* Moved load-test and producer to tools package

* Updated Spark Script and added server README

* Updated Spark Script and added server README

* Improve client tests and cover output messages

* Remove server conf from flient files

* Fixing CI

* Fixing Ci

* Remove docker image cache

* Build dev server img with build client image

* Fix server tests

* Test docker command with tty

* Fix server tests CI

* Fix Ci docker push

* Remove unecessary Client Requests Metrics from server

* Remove Server metrics from client

.dockerignore

@@ -1,2 +1,2 @@
 bin
-vendor
+vendor

.github/workflows/docker-image.yml

@@ -7,10 +7,6 @@ jobs:
     name: Push Docker image to Docker Hub
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v1
-        with:
-          go-version: '1.17'
       - name: Check out the repo
         uses: actions/checkout@v4
       - name: Set up QEMU
@@ -34,3 +30,5 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           platforms: linux/amd64,linux/arm64
+          file: server/Dockerfile
+          context: server/

.github/workflows/test.yml

@@ -1,18 +1,19 @@
 name: Run tests
-on:
-  push:
+on: push
 jobs:
-  tests:
-    name: Run unit and integration tests
+  client-tests:
+    name: Run client tests
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.17'
       - name: Check out the repo
-        uses: actions/checkout@v2
-      - name: Set up ci
-        run: make setup-ci
-      - name: Test
-        run: GOPATH=/home/runner/go make test
+        uses: actions/checkout@v4
+      - run: make build-dev
+      - run: make test
+  server-tests:
+    name: Run server tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+      - run: cd server && make build-dev
+      - run: cd server && make test

.gitignore

@@ -21,7 +21,8 @@ bin
 *.swp
 
 ./eventsgateway
-spark-notebook/work/.ipynb_checkpoints
+spark-notebook/.ipynb_checkpoints
+spark-notebook/spark-warehouse
 
 Sidecarfile
 

Makefile

@@ -6,134 +6,87 @@
 # Copyright © 2018 Top Free Games <backend@tfgco.com>
 
 MY_IP=`ifconfig | grep --color=none -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep --color=none -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1' | head -n 1`
-TEST_PACKAGES=`find . -type f -name "*.go" ! \( -path "*vendor*" \) | sed -En "s/([^\.])\/.*/\1/p" | uniq`
+TEST_PACKAGES=`find . -type f -name "*.go" ! \( -path "*server*" \) | sed -En "s/([^\.])\/.*/\1/p" | uniq`
 GOBIN="${GOPATH}/bin"
 
 .PHONY: load-test producer spark-notebook
 
-setup: setup-hooks
-	@go install github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad
-	@go install github.com/onsi/ginkgo/v2/ginkgo@v2.1.4
-	@go mod tidy
-
-setup-hooks:
-	@cd .git/hooks && ln -sf ./hooks/pre-commit.sh pre-commit
-
-setup-ci:
-	@go install github.com/mattn/goveralls@v0.0.11
-	@go install github.com/onsi/ginkgo/v2/ginkgo@v2.1.4
-	@go install github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad
-	@go mod tidy
+build-dev:
+	@docker build -t eventsgateway-client-dev -f dev.Dockerfile .
+	@make -f server/Makefile build-dev
 
-build:
-	@mkdir -p bin && go build -o ./bin/eventsgateway main.go
-
-build-docker:
-	@docker build -t eventsgateway .
-
-deps-start:
-	@echo "Starting dependencies using HOST IP of ${MY_IP}..."
-	@env MY_IP=${MY_IP} docker compose --project-name eventsgateway up -d \
-		zookeeper kafka localstack jaeger
-	@echo "Dependencies started successfully."
-
-deps-stop:
-	@env MY_IP=${MY_IP} docker compose --project-name eventsgateway down
-
-deps-test-start:
-	@env MY_IP=${MY_IP} docker compose --project-name eventsgateway-test up -d \
-		zookeeper kafka jaeger
-
-deps-test-stop:
-	@env MY_IP=${MY_IP} docker compose --project-name eventsgateway-test down
-
-cross-build-linux-amd64:
-	@env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o ./bin/eventsgateway-linux-amd64
-	@chmod a+x ./bin/eventsgateway-linux-amd64
-
-run:
-	@echo "Will connect to kafka at ${MY_IP}:9192"
-	@echo "OTLP exporter endpoint at http://${MY_IP}:4317"
-	@env EVENTSGATEWAY_KAFKA_PRODUCER_BROKERS=${MY_IP}:9192 OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://${MY_IP}:4317 go run main.go start -d
+test:
+	docker compose up client-tests
 
+spark-notebook:
+	@docker compose up jupyter
 
 producer:
 	@echo "Will connect to server at ${MY_IP}:5000"
-	@env EVENTSGATEWAY_PROMETHEUS_PORT=9092 go run main.go producer -d
+	@go run main.go producer -d
 
 load-test:
 	@echo "Will connect to server at ${MY_IP}:5000"
-	@env EVENTSGATEWAY_PROMETHEUS_PORT=9092 go run main.go load-test -d
-
-spark-notebook:
-	@env MY_IP=${MY_IP} docker compose --project-name eventsgateway up -d \
-		spark-notebook
+	@go run main.go load-test -d
 
-hive-start:
-	@echo "Starting Hive stack using HOST IP of ${MY_IP}..."
-	@cd ./hive && docker compose up
-	@echo "Hive stack started successfully."
+deps-start:
+	@docker compose up -d eventsgateway-api
 
-hive-stop:
-	@cd ./hive && docker compose down
+setup:
+	@go install github.com/onsi/ginkgo/v2/ginkgo@v2.1.4
+	@go install github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad
+	@go mod tidy
+	@cd .git/hooks && ln -sf ./hooks/pre-commit.sh pre-commit
 
-unit: unit-board clear-coverage-profiles unit-run gather-unit-profiles
+setup-ci:
+	@go install github.com/mattn/goveralls@v0.0.11
+	@go install github.com/onsi/ginkgo/v2/ginkgo@v2.1.4
+	@go install github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad
+	@go mod tidy
 
-clear-coverage-profiles:
-	@find . -name '*.coverprofile' -delete
+test-go: unit integration test-coverage-func
 
-unit-board:
-	@echo
-	@echo "\033[1;34m=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\033[0m"
-	@echo "\033[1;34m=         Unit Tests         -\033[0m"
-	@echo "\033[1;34m=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\033[0m"
+unit: print-unit-section unit-run copy-unit-cover
 
 unit-run:
-	@${GOBIN}/ginkgo -tags unit -cover -r --randomize-all --randomize-suites ${TEST_PACKAGES}
+	@${GOBIN}/ginkgo -tags unit -v -cover --covermode count -r --randomize-all --randomize-suites ${TEST_PACKAGES}
 
-gather-unit-profiles:
-	@mkdir -p _build
-	@echo "mode: count" > _build/coverage-unit.out
-	@bash -c 'for f in $$(find . -name "*.coverprofile"); do tail -n +2 $$f >> _build/coverage-unit.out; done'
-
-int integration: integration-board clear-coverage-profiles deps-test-start integration-run gather-integration-profiles deps-test-stop
-
-integration-board:
-	@echo
-	@echo "\033[1;34m=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\033[0m"
-	@echo "\033[1;34m=     Integration Tests      -\033[0m"
-	@echo "\033[1;34m=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\033[0m"
+integration: print-integration-section integration-run copy-integration-cover
 
 integration-run:
-	@GRPC_GO_RETRY=on OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://${MY_IP}:4317 ${GOBIN}/ginkgo -tags integration -cover -r --randomize-all --randomize-suites \
+	@GRPC_GO_RETRY=on ${GOBIN}/ginkgo -v -tags integration -cover --covermode count -r --randomize-all --randomize-suites \
 		--skip-package=./app ${TEST_PACKAGES}
 
-int-ci: integration-board clear-coverage-profiles deps-test-ci integration-run gather-integration-profiles
+copy-unit-cover:
+	@mkdir -p _build
+	@cp ./coverprofile.out _build/coverage-unit.out
 
-gather-integration-profiles:
+copy-integration-cover:
 	@mkdir -p _build
-	@echo "mode: count" > _build/coverage-integration.out
-	@bash -c 'for f in $$(find . -name "*.coverprofile"); do tail -n +2 $$f >> _build/coverage-integration.out; done'
+	@cp ./coverprofile.out _build/coverage-integration.out
 
 merge-profiles:
 	@mkdir -p _build
 	@${GOBIN}/gocovmerge _build/*.out > _build/coverage-all.out
 
-test-coverage-func coverage-func: merge-profiles
+print-unit-section:
 	@echo
-	@echo "\033[1;34m=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\033[0m"
-	@echo "\033[1;34mFunctions NOT COVERED by Tests\033[0m"
-	@echo "\033[1;34m=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\033[0m"
-	@go tool cover -func=_build/coverage-all.out | egrep -v "100.0[%]"
+	@echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+	@echo "=        Unit Tests         ="
+	@echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
 
-test: unit int test-coverage-func
+print-integration-section:
+	@echo
+	@echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+	@echo "=     Integration Tests     ="
+	@echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
 
-test-ci: unit test-coverage-func
+test-coverage-func: merge-profiles
+	@echo
+	@echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+	@echo "Functions NOT COVERED by Tests  ="
+	@echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+	@go tool cover -func=_build/coverage-all.out | egrep -v "100.0[%]"
 
 test-coverage-html cover:
 	@go tool cover -html=_build/coverage-all.out
-
-rtfd:
-	@rm -rf docs/_build
-	@sphinx-build -b html -d ./docs/_build/doctrees ./docs/ docs/_build/html
-	@open docs/_build/html/index.html

README.md

@@ -78,7 +78,9 @@ func main() {
 
 All dependencies required to produce and consume events locally are bundled in this project.
 
-1. `make deps-start` will start docker containers for `zookeeper` `kafka` and `localstack`.
+1. `make build-dev` will build a development docker image for the client.
+2. `make deps-start` will start docker containers for `zookeeper` `kafka`, `eventsgateway-api`
+2. 
 
 These are the necessary dependencies for EventsGateway server.