Sidecar to Daemonset (#97)

* initial commit

* updates

* first attempt in allocation

* adding some unit tests

* use sync.map

* updates per PR review

* updated diagram

* removing mentions of sidecar from the docs

* adding liveness probe

* adding some handling for when the DaemonSet crashes

* updates on NodeAgent code

* fixing tests

.github/workflows/main.yml

@@ -34,10 +34,10 @@ jobs:
  run: cd tools/gameserverapi && GIN_MODE=release go test
  - name: initcontainer unit tests
  run: cd initcontainer && go test
- - name: sidecar unit tests
- run: cd sidecar-go && go test
+ - name: nodeagent unit tests
+ run: cd nodeagent && go test
  - name: operator unit tests
- run: IMAGE_NAME_SAMPLE=thundernetes-netcore-sample IMAGE_NAME_INIT_CONTAINER=thundernetes-initcontainer IMAGE_NAME_SIDECAR=thundernetes-sidecar-go TAG=$(git rev-list HEAD --max-count=1 --abbrev-commit) make -C operator test
+ run: IMAGE_NAME_SAMPLE=thundernetes-netcore-sample IMAGE_NAME_INIT_CONTAINER=thundernetes-initcontainer TAG=$(git rev-list HEAD --max-count=1 --abbrev-commit) make -C operator test
  - name: install kind binaries
  run: make installkind
  - name: create kind cluster

.versions

@@ -1,6 +1,6 @@
 #!/bin/bash
 export OPERATOR_TAG=0.1.0
+export NODE_AGENT_TAG=0.1.0
 export INIT_CONTAINER_TAG=0.1.0
-export SIDECAR_TAG=0.1.0
 export NETCORE_SAMPLE_TAG=0.1.0
 export OPENARENA_SAMPLE_TAG=0.1.0

Makefile

@@ -1,14 +1,14 @@
 NS ?= ghcr.io/playfab
 
 export IMAGE_NAME_OPERATOR=thundernetes-operator
+export IMAGE_NAME_NODE_AGENT=thundernetes-nodeagent
 export IMAGE_NAME_INIT_CONTAINER=thundernetes-initcontainer
-export IMAGE_NAME_SIDECAR=thundernetes-sidecar-go
 export IMAGE_NAME_NETCORE_SAMPLE=thundernetes-netcore-sample
 export IMAGE_NAME_OPENARENA_SAMPLE=thundernetes-openarena-sample
 
 export OPERATOR_TAG?=$(shell git rev-list HEAD --max-count=1 --abbrev-commit)
+export NODE_AGENT_TAG?=$(shell git rev-list HEAD --max-count=1 --abbrev-commit)
 export INIT_CONTAINER_TAG?=$(shell git rev-list HEAD --max-count=1 --abbrev-commit)
-export SIDECAR_TAG?=$(shell git rev-list HEAD --max-count=1 --abbrev-commit)
 export NETCORE_SAMPLE_TAG?=$(shell git rev-list HEAD --max-count=1 --abbrev-commit)
 export OPENARENA_SAMPLE_TAG?=$(shell git rev-list HEAD --max-count=1 --abbrev-commit)
 
@@ -20,28 +20,24 @@ export KIND_CLUSTER_NAME=kind
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
-docker-compose:
- rm -rf samples/data/GameLogs/*
- docker-compose up --build
-
 build:
  docker build -f ./operator/Dockerfile -t $(NS)/$(IMAGE_NAME_OPERATOR):$(OPERATOR_TAG) ./operator
+ docker build -f ./nodeagent/Dockerfile -t $(NS)/$(IMAGE_NAME_NODE_AGENT):$(NODE_AGENT_TAG) ./nodeagent
  docker build -f ./initcontainer/Dockerfile -t $(NS)/$(IMAGE_NAME_INIT_CONTAINER):$(INIT_CONTAINER_TAG) ./initcontainer
- docker build -f ./sidecar-go/Dockerfile -t $(NS)/$(IMAGE_NAME_SIDECAR):$(SIDECAR_TAG) ./sidecar-go
  docker build -f ./samples/netcore/Dockerfile -t $(NS)/$(IMAGE_NAME_NETCORE_SAMPLE):$(NETCORE_SAMPLE_TAG) ./samples/netcore
  docker build -f ./samples/openarena/Dockerfile -t $(NS)/$(IMAGE_NAME_OPENARENA_SAMPLE):$(OPENARENA_SAMPLE_TAG) ./samples/openarena
-
+ 
 push:
  docker push $(NS)/$(IMAGE_NAME_OPERATOR):$(OPERATOR_TAG)
+ docker push $(NS)/$(IMAGE_NAME_NODE_AGENT):$(NODE_AGENT_TAG)
  docker push $(NS)/$(IMAGE_NAME_INIT_CONTAINER):$(INIT_CONTAINER_TAG)
- docker push $(NS)/$(IMAGE_NAME_SIDECAR):$(SIDECAR_TAG)
  docker push $(NS)/$(IMAGE_NAME_NETCORE_SAMPLE):$(NETCORE_SAMPLE_TAG)
  docker push $(NS)/$(IMAGE_NAME_OPENARENA_SAMPLE):$(OPENARENA_SAMPLE_TAG)
 
 builddockerlocal:
- docker build -f operator/Dockerfile -t $(IMAGE_NAME_OPERATOR):$(OPERATOR_TAG) ./operator 
+ docker build -f operator/Dockerfile -t $(IMAGE_NAME_OPERATOR):$(OPERATOR_TAG) ./operator
+ docker build -f nodeagent/Dockerfile -t $(IMAGE_NAME_NODE_AGENT):$(NODE_AGENT_TAG) ./nodeagent
  docker build -f initcontainer/Dockerfile -t $(IMAGE_NAME_INIT_CONTAINER):$(INIT_CONTAINER_TAG) ./initcontainer 
- docker build -f sidecar-go/Dockerfile -t $(IMAGE_NAME_SIDECAR):$(SIDECAR_TAG) ./sidecar-go 
  docker build -f samples/netcore/Dockerfile -t $(IMAGE_NAME_NETCORE_SAMPLE):$(NETCORE_SAMPLE_TAG) ./samples/netcore 
  docker build -f samples/openarena/Dockerfile -t $(IMAGE_NAME_OPENARENA_SAMPLE):$(OPENARENA_SAMPLE_TAG) ./samples/openarena 
 
@@ -74,9 +70,9 @@ create-install-files:
  . .versions && \
  IMG=$(NS)/$(IMAGE_NAME_OPERATOR):$${OPERATOR_TAG} \
  IMAGE_NAME_INIT_CONTAINER=$(NS)/$(IMAGE_NAME_INIT_CONTAINER) \
- IMAGE_NAME_SIDECAR=$(NS)/$(IMAGE_NAME_SIDECAR) \
- SIDECAR_TAG=$${SIDECAR_TAG} \
+ IMAGE_NAME_NODE_AGENT=$(NS)/$(IMAGE_NAME_NODE_AGENT) \
  INIT_CONTAINER_TAG=$${INIT_CONTAINER_TAG} \
+ NODE_AGENT_TAG=$${NODE_AGENT_TAG} \
  make -C operator create-install-files
 
 create-install-files-dev:

docs/FAQ.md

@@ -66,38 +66,9 @@ Scaling in Kubernetes is two fold. Pod autoscaling and Cluster autoscaling. Thun
 
 In conjuction with cluster autoscaler, you can use [Virtual Kubelet](https://github.com/virtual-kubelet/virtual-kubelet) project to accelerate the addition of new Pods to the cluster. If you are using Azure Kubernetes Service, you can easily enable Virtual Nodes feature (which is based on Virtual Kubelet) using the instructions [here](https://docs.microsoft.com/en-us/azure/aks/virtual-nodes).
 
-## How can I run my game server pods in a non-default namespace?
-
-By default, thundernetes monitors the `default` namespace. If you want to run your game servers in a different namespace, you should first install the necessary ServiceAccount/RoleBinding RBAC roles on this namespace. This is because the sidecar running on the GameServer Pod needs access to talk to the Kubernetes API server. For information on Kubernetes RBAC, check [here](https://kubernetes.io/docs/reference/access-authn-authz/rbac/).
-
-You can save the following configuration on a yaml file and then run `kubectl apply -f /path/to/file.yaml` to create the namespace and RBAC objects
-
-```yaml
-apiVersion: v1
-kind: Namespace
-metadata:
- name: mynamespace
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: gameserver-editor
- namespace: mynamespace
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
- name: gameserver-editor-rolebinding
- namespace: mynamespace
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: gameserver-editor-role
-subjects:
-- kind: ServiceAccount
- name: gameserver-editor
- namespace: mynamespace 
-```
+## Can I run my game server pods in a non-default namespace?
+
+You don't need to anything special to run your game server Pods in a namespace different than the "default". Old versions of thundernetes (up to 0.1) made use of a sidecar to access the Kubernetes API Server, so you need to create special RoleBinding and ServiceAccount in your namespace. With the transition to DaemonSet NodeAgent in 0.2, this is no longer necessary.
 
 ## How do I schedule thundernetes Pods and GameServer Pods into different Nodes?
 

docs/architecture.md

@@ -18,7 +18,7 @@ Specifically, we have two core entities in our project, which are represented by
 
 ## GSDK integration
 
-We have created a [sidecar](https://www.magalix.com/blog/the-sidecar-pattern) container that receives all the GSDK calls and modifies the GameServer state accordingly. The sidecar container is automatically created by thundernetes and runs on the same Pod as the GameServer container. It is also responsible for "watching" (via a Kubernetes watch) the state of the GameServer object, getting a notification when it changes. This is useful to track when the GameServer has been allocated. You can find the GSDK source code [here](https://github.com/PlayFab/gsdk) and check the docs [here](https://docs.microsoft.com/en-us/gaming/playfab/features/multiplayer/servers/integrating-game-servers-with-gsdk). The GSDK is used by game servers running in Azure PlayFab Multiplayer Servers (MPS), thus making the migration from thundernetes to MPS (and vice versa) pretty seamless. To verify GSDK integration, you can use [LocalMultiplayerAgent](https://github.com/PlayFab/MpsAgent).
+We have created a [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) Pod that runs on all Nodes in the cluster, called NodeAgent.NodeAgent sets up a web server that receives all the GSDK calls from the GameServer Pods on each Node and modifies the GameServer state accordingly. It is also responsible for "watching" (via a Kubernetes watch) the state of the GameServer objects that reside on the same Node, getting a notification when it changes. This is useful to track when the GameServer has been allocated. You can find the GSDK source code [here](https://github.com/PlayFab/gsdk) and check the docs [here](https://docs.microsoft.com/en-us/gaming/playfab/features/multiplayer/servers/integrating-game-servers-with-gsdk). The GSDK is used by game servers running in Azure PlayFab Multiplayer Servers (MPS), thus making the migration from thundernetes to MPS (and vice versa) pretty seamless. To verify GSDK integration, you can use [LocalMultiplayerAgent](https://github.com/PlayFab/MpsAgent).
 
 ### initcontainer
 
@@ -53,13 +53,15 @@ Noteworthy is the fact that this port range is used for all GameServerBuilds and
 When you allocate a GameServer, thundernetes needs to do two things:
 
 - Find a GameServer instance for the requested GameServerBuild in the StandindBy state and update it to the Active state.
-- Inform the corresponding GameServer Pod (specifically, the sidecar container in that Pod) that the GameServer state is now Active. The sidecar will give this information back to the GameServer container. The way that this is accomplished is the following: each GameServer process/container regularly heartbeats (sends a JSON HTTP request) to the sidecar. When the sidecar is notified that the GameServer state has transitioned to Active, it will respond with the new state to the heartbeat coming from the GameServer container.
+- Inform the corresponding GameServer Pod (specifically, the NodeAgent that handles GSDK calls for that Pod) that the GameServer state is now Active. NodeAgent will give this information back to the GameServer container. The way that this is accomplished is the following: each GameServer process/container regularly heartbeats (sends a JSON HTTP request) to the NodeAgent. When the NodeAgent is notified that the GameServer state has transitioned to Active, it will respond with the new state to the heartbeat coming from the GameServer container.
 
 There are two ways we can accomplish the second step:
 
-- Have a [Kubernetes watch](https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes) from the sidecar to the Kubernetes' API server which will be notified when the GameServer is updated. This approach works well from a security perspective, since you can configure RBAC rules for the GameServer Pod.
-- Have the controller's allocation API service forward the allocation request to the sidecar. This is done via having the sidecar expose its HTTP server inside the cluster. Of course, this assumes that we trust the processes running on the containers in the cluster.
+- Have a [Kubernetes watch](https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes) from the NodeAgent to the Kubernetes' API server which will be notified when the GameServer is updated. This approach works well from a security perspective, since you can configure RBAC rules for the GameServer Pod.
+- Have the controller's allocation API service forward the allocation request to the NodeAgent. This is done via having the NodeAgent expose its HTTP server inside the cluster. Of course, this assumes that we trust the processes running on the containers in the cluster.
 
-For communicating with the sidecar, we eventually picked the first approach. The second approach was used initially but was abandoned due to security concerns.
+For communicating with the NodeAgent, we eventually picked the first approach. The second approach was used initially but was abandoned due to security concerns.
 
 Moreover, when the user allocates a GameServer, we create an instance of the GameServerDetail CR which stores the InitialPlayers (if any). The GameServerDetail has a 1:1 relationship with the GameServer CR and share the same name. Moreover, GameServer is the owner of the GameServerDetail instance so both will be deleted, upon GameServer's deletion. The GameServerDetail also tracks the ConnectedPlayersCount and ConnectedPlayers names/IDs, which can be updated by the UpdateConnectedPlayers GSDK method.
+
+Worth mentioning here is the fact that up to 0.1, the NodeAgent process was a sidecar container that lived inside the GameServer Pod. However, on version 0.2 we transitioned to a NodeAgent Pod that runs on all Nodes in the cluster. This is done to avoid the need for a sidecar container.

docs/development.md

@@ -33,18 +33,6 @@ kubectl create namespace thundernetes-system
 kubectl create secret tls tls-secret -n thundernetes-system --cert=/home/dgkanatsios/public.pem --key=/home/dgkanatsios/private.pem
 ```
 
-### Install/deploy a specific commit
-
-```bash
-export TAG=$(git rev-list HEAD --max-count=1 --abbrev-commit)
-make build push
-IMG=ghcr.io/playfab/thundernetes-operator:${TAG} \
- IMAGE_NAME_INIT_CONTAINER=docker.io/dgkanatsios/thundernetes-initcontainer \
- IMAGE_NAME_SIDECAR=docker.io/dgkanatsios/thundernetes-sidecar-go \
- API_SERVICE_SECURITY=none \
- make -C operator install deploy
-```
-
 ### Allocate a game server
 
 #### With TLS auth
@@ -87,7 +75,7 @@ make deletekindcluster && make builddockerlocal && make createkindcluster && mak
 
 ```bash
 cd operator
-THUNDERNETES_SIDECAR_IMAGE=ghcr.io/playfab/thundernetes-sidecar-go:0.1.0 THUNDERNETES_INIT_CONTAINER_IMAGE=ghcr.io/playfab/thundernetes-initcontainer:0.1.0 go run main.go
+THUNDERNETES_INIT_CONTAINER_IMAGE=ghcr.io/playfab/thundernetes-initcontainer:0.1.0 go run main.go
 ```
 
 ## [ADVANCED] Install thundernetes via cloning this repository

docs/troubleshooting/GameServers/README.md

@@ -6,5 +6,4 @@ This contains troubleshooting troubleshooting Game Servers related to thundernet
 * [How can I find the public IP Address from inside a Game Server?](./HowCanIFindThePublicIPAddress.md)
 * [Help! My Game server is stuck!](./MyGameServerIsStuck.md)
 
-* [My Game Servre is stuck. What can I do?](./MyGameServerIsStuck.md)
-* [How can I run my game server pods in a non-default namespace?](./HowCanIRunMyGameServerPodInANonDefaultNamespace.md)
+* [My Game Server is stuck. What can I do?](./MyGameServerIsStuck.md)