MF-643 - Create end-to-end tests for openAPI

.github/workflows/st.yaml

@@ -0,0 +1,181 @@
+name: Test OpenAPI using Schemathesis
+
+on: [push, pull_request] 
+
+env:
+  MF_TOKEN: abcd
+  EMAIL: admin@example.com
+  PASSWORD: 12345678
+  DEVICE: mf-device
+
+jobs:
+  api-tests:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: checkout repo
+        uses: actions/checkout@v2
+
+      - name: find number of cores
+        run: lscpu|egrep 'Model name|Socket|Thread|NUMA|CPU\(s\)'
+
+      - name: dockerversion
+        run: docker-compose --version
+
+      # - name: update docker-compose
+      #   run: sudo apt-get update && sudo apt-get install docker-compose
+
+      # - name: Install docker-compose
+      #   get_url:
+      #     url : https://github.com/docker/compose/releases/download/1.26.2/docker-compose-Linux-x86_64
+      #     dest: /usr/local/bin/docker-compose
+      #     mode: '755'
+      #     force: yes  # Handle GitHub returning 304
+      #   become: yes
+
+      - name: Uninstall old versions of Docker and Docker Compose
+        run: |
+          sudo apt-get remove docker docker-engine docker.io containerd runc
+          sudo rm /usr/local/bin/docker-compose
+      - name: Install Docker
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+          sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+          sudo apt-get update
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+      - name: Install Docker Compose
+        run: |
+          sudo curl -L "https://github.com/docker/compose/releases/download/$(curl --silent https://api.github.com/repos/docker/compose/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+
+
+      - name: updated dockerversion
+        run: docker-compose --version
+
+      - name: create images
+        if: always()
+        run: make all -j 2 && make -j 2 dockers_dev
+
+      - name: run containers
+        if: always()
+        run: sudo docker-compose -f docker/docker-compose.yml --env-file docker/.env up -d
+
+      - name: REST API with curl to provision user
+        if: always()
+        run: |
+          curl -sSiX POST http://localhost:9002/users -H "Content-Type: application/json" -d '{"name": "John Doe","credentials": {"identity": "admin@example.com","secret": "12345678"}}'
+
+      - name: Set ACCESS_TOKEN
+        id: set-access-token
+        run: |
+          response='{"access_token":"..." ... }' # Replace with your JSON response
+          access_token=$(echo "$response" | grep -o '"access_token":"[^"]*' | cut -d'"' -f4)
+          echo "ACCESS_TOKEN=$access_token" >> $GITHUB_ENV
+
+      - name: Set BEARER_TOKEN
+        run: echo "BEARER_TOKEN=rer $MF_TOKEN" >> $GITHUB_ENV
+
+      - name: print bearer token
+        if: always()
+        run: echo $BEARER_TOKEN
+
+      - name: Set token env variable
+        run: echo "MF_TOKEN=$(echo $MF_TOKEN)" >> $GITHUB_ENV
+
+      - name: Test
+        run: echo ${{ env.ACCESS_TOKEN }}
+
+      # Bootstrap.yml
+      - name: Run st for bootstrap.yml
+        if: always()
+        uses: schemathesis/action@v1
+        with:
+          schema: './api/openapi/bootstrap.yml'
+          base-url: 'http://localhost'
+          args: '-H "Authorization: Bearer {{ env.ACCESS_TOKEN }}" --hypothesis-suppress-health-check=filter_too_much,too_slow'
+          token: ${{ secrets.SCHEMATHESIS_TOKEN }}
+
+      # Certs.yml
+      - name: Run st for certs.yml
+        if: always()
+        uses: schemathesis/action@v1
+        with:
+          schema: './api/openapi/certs.yml'
+          base-url: 'http://localhost'
+          args: '-H "Authorization: Bearer {{ env.ACCESS_TOKEN }}" --hypothesis-suppress-health-check=filter_too_much,too_slow'
+          token: ${{ secrets.SCHEMATHESIS_TOKEN }}
+
+      # Consumers-notifiers.yml
+      - name: Run st for consumers-notifiers.yml
+        if: always()
+        uses: schemathesis/action@v1
+        with:
+          schema: './api/openapi/consumers-notifiers.yml'
+          base-url: 'http://localhost'
+          args: '-H "Authorization: Bearer {{ env.ACCESS_TOKEN }}" --hypothesis-suppress-health-check=filter_too_much,too_slow'
+          token: ${{ secrets.SCHEMATHESIS_TOKEN }}
+
+      # Http.yml
+      - name: Run st for http.yml
+        if: always()
+        uses: schemathesis/action@v1
+        with:
+          schema: './api/openapi/http.yml'
+          base-url: 'http://localhost'
+          args: '-H "Authorization: Bearer {{ env.ACCESS_TOKEN }}" --hypothesis-suppress-health-check=filter_too_much,too_slow'
+          token: ${{ secrets.SCHEMATHESIS_TOKEN }}
+
+      # Provision.yml
+      - name: Run st for provision.yml
+        if: always()
+        uses: schemathesis/action@v1
+        with:
+          schema: './api/openapi/provision.yml'
+          base-url: 'http://localhost'
+          args: '-H "Authorization: Bearer {{ env.ACCESS_TOKEN }}" --hypothesis-suppress-health-check=filter_too_much,too_slow'
+          token: ${{ secrets.SCHEMATHESIS_TOKEN }}
+
+      # Readers.yml
+      - name: Run st for readers.yml
+        if: always()
+        uses: schemathesis/action@v1
+        with:
+          schema: './api/openapi/readers.yml'
+          base-url: 'http://localhost'
+          args: '-H "Authorization: Bearer {{ env.ACCESS_TOKEN }}" --hypothesis-suppress-health-check=filter_too_much,too_slow'
+          token: ${{ secrets.SCHEMATHESIS_TOKEN }}
+
+      # Things.yml
+      - name: Run st for things.yml
+        if: always()
+        uses: schemathesis/action@v1
+        with:
+          schema: './api/openapi/things.yml'
+          base-url: 'http://localhost'
+          args: '-H "Authorization: Bearer {{ env.ACCESS_TOKEN }}" --hypothesis-suppress-health-check=filter_too_much,too_slow'
+          token: ${{ secrets.SCHEMATHESIS_TOKEN }}
+
+      # Twins.yml
+      - name: Run st for twins.yml
+        if: always()
+        uses: schemathesis/action@v1
+        with:
+          schema: './api/openapi/twins.yml'
+          base-url: 'http://localhost'
+          args: '-H "Authorization: Bearer {{ env.ACCESS_TOKEN }}" --hypothesis-suppress-health-check=filter_too_much,too_slow'
+          token: ${{ secrets.SCHEMATHESIS_TOKEN }}
+
+      # Users.yml
+      - name: Run st for users.yml
+        if: always()
+        uses: schemathesis/action@v1
+        with:
+          schema: './api/openapi/users.yml'
+          base-url: 'http://localhost'
+          args: '-H "Authorization: Bearer {{ env.ACCESS_TOKEN }}" --hypothesis-suppress-health-check=filter_too_much,too_slow'
+          token: ${{ secrets.SCHEMATHESIS_TOKEN }}
+
+      - name: Stop the running docker containers
+        if: always()
+        run: make stop 

.github/workflows/swagger-ui.yaml

@@ -24,4 +24,4 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: swagger-ui
           cname: api.mainflux.io
-
+          

Makefile

@@ -13,14 +13,7 @@ GOARCH ?= amd64
 VERSION ?= $(shell git describe --abbrev=0 --tags)
 COMMIT ?= $(shell git rev-parse HEAD)
 TIME ?= $(shell date +%F_%T)
-USER_REPO ?= $(shell git remote get-url origin | sed -e 's/.*\/\([^/]*\)\/\([^/]*\).*/\1_\2/' )
-BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD  2>/dev/null || git describe --tags --abbrev=0  2>/dev/null )
-empty:=
-space:= $(empty) $(empty)
-DOCKER_PROJECT ?= $(shell echo $(subst $(space),,$(USER_REPO)_$(BRANCH)) | tr -c -s '[:alnum:][=-=]' '_')
-DOCKER_COMPOSE_COMMANDS_SUPPORTED := up down config
-DEFAULT_DOCKER_COMPOSE_COMMAND  := up
-GRPC_MTLS_CERT_FILES_EXISTS = 0
+
 ifneq ($(MF_BROKER_TYPE),)
     MF_BROKER_TYPE := $(MF_BROKER_TYPE)
 else
@@ -61,36 +54,9 @@ define make_docker_dev
 		-f docker/Dockerfile.dev ./build
 endef
 
-ADDON_SERVICES = bootstrap cassandra-reader cassandra-writer certs \
-					influxdb-reader influxdb-writer lora-adapter mongodb-reader mongodb-writer \
-					opcua-adapter postgres-reader postgres-writer provision smpp-notifier smtp-notifier \
-					timescale-reader timescale-writer twins
-
-EXTERNAL_SERVICES = vault prometheus
-
-ifneq ($(filter run%,$(firstword $(MAKECMDGOALS))),)
-  temp_args := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
-  DOCKER_COMPOSE_COMMAND := $(if $(filter $(DOCKER_COMPOSE_COMMANDS_SUPPORTED),$(temp_args)), $(filter $(DOCKER_COMPOSE_COMMANDS_SUPPORTED),$(temp_args)), $(DEFAULT_DOCKER_COMPOSE_COMMAND))
-  $(eval $(DOCKER_COMPOSE_COMMAND):;@)
-endif
-
-ifneq ($(filter run_addons%,$(firstword $(MAKECMDGOALS))),)
-  temp_args := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
-  RUN_ADDON_ARGS :=  $(if $(filter-out $(DOCKER_COMPOSE_COMMANDS_SUPPORTED),$(temp_args)), $(filter-out $(DOCKER_COMPOSE_COMMANDS_SUPPORTED),$(temp_args)),$(ADDON_SERVICES) $(EXTERNAL_SERVICES))
-  $(eval $(RUN_ADDON_ARGS):;@)
-endif
-
-ifneq ("$(wildcard docker/ssl/certs/*-grpc-*)","")
-GRPC_MTLS_CERT_FILES_EXISTS = 1
-else
-GRPC_MTLS_CERT_FILES_EXISTS = 0
-endif
-
-FILTERED_SERVICES = $(filter-out $(RUN_ADDON_ARGS), $(SERVICES))
-
 all: $(SERVICES)
 
-.PHONY: all $(SERVICES) dockers dockers_dev latest release run run_addons grpc_mtls_certs check_mtls check_certs
+.PHONY: all $(SERVICES) dockers dockers_dev latest release
 
 clean:
 	rm -rf ${BUILD_DIR}
@@ -115,7 +81,7 @@ proto:
 	protoc -I. --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative users/policies/*.proto
 	protoc -I. --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative things/policies/*.proto
 
-$(FILTERED_SERVICES):
+$(SERVICES):
 	$(call compile_service,$(@))
 
 $(DOCKERS):
@@ -151,45 +117,10 @@ release:
 rundev:
 	cd scripts && ./run.sh
 
-grpc_mtls_certs:
-	$(MAKE) -C docker/ssl users_grpc_certs things_grpc_certs
-
-check_tls:
-ifeq ($(GRPC_TLS),true)
-	@unset GRPC_MTLS
-	@echo "gRPC TLS is enabled"
-	GRPC_MTLS=
-else
-	@unset GRPC_TLS
-	GRPC_TLS=
-endif
-
-check_mtls:
-ifeq ($(GRPC_MTLS),true)
-	@unset GRPC_TLS
-	@echo "gRPC MTLS is enabled"
-	GRPC_TLS=
-else
-	@unset GRPC_MTLS
-	GRPC_MTLS=
-endif
-
-check_certs: check_mtls check_tls
-ifeq ($(GRPC_MTLS_CERT_FILES_EXISTS),0)
-ifeq ($(filter true,$(GRPC_MTLS) $(GRPC_TLS)),true)
-ifeq ($(filter $(DEFAULT_DOCKER_COMPOSE_COMMAND),$(DOCKER_COMPOSE_COMMAND)),$(DEFAULT_DOCKER_COMPOSE_COMMAND))
-	$(MAKE) -C docker/ssl users_grpc_certs things_grpc_certs
-endif
-endif
-endif
-
-run: check_certs
+run:
 	sed -i "s,file: brokers/.*.yml,file: brokers/${MF_BROKER_TYPE}.yml," docker/docker-compose.yml
 	sed -i "s,MF_BROKER_URL=.*,MF_BROKER_URL=$$\{MF_$(shell echo ${MF_BROKER_TYPE} | tr 'a-z' 'A-Z')_URL\}," docker/.env
-	docker-compose -f docker/docker-compose.yml -p $(DOCKER_PROJECT) $(DOCKER_COMPOSE_COMMAND) $(args)
+	docker-compose -f docker/docker-compose.yml up
 
-run_addons: check_certs
-	$(foreach SVC,$(RUN_ADDON_ARGS),$(if $(filter $(SVC),$(ADDON_SERVICES) $(EXTERNAL_SERVICES)),,$(error Invalid Service $(SVC))))
-	@for SVC in $(RUN_ADDON_ARGS); do \
-		MF_ADDONS_CERTS_PATH_PREFIX="../."  docker-compose -f docker/addons/$$SVC/docker-compose.yml -p $(DOCKER_PROJECT) --env-file ./docker/.env $(DOCKER_COMPOSE_COMMAND) $(args) & \
-	done
+stop:
+	docker-compose -f docker/docker-compose.yml down -v