From 3b34915e0c573f1e1bc1aa83906e55afde37e77c Mon Sep 17 00:00:00 2001 From: Dejan Pejchev Date: Fri, 3 Nov 2023 14:18:07 +0100 Subject: [PATCH 1/3] add better guides for quickstart --- README.md | 75 +++++++++---------------------- dev/quickstart/README.md | 9 ++++ dev/quickstart/armada-crds.yaml | 2 +- dev/quickstart/job.yaml | 22 +++++++++ dev/quickstart/minimal-setup.yaml | 57 +++++++++++++++++++++++ dev/runbooks/0-dependencies.md | 46 +++++++++++++++++++ dev/runbooks/1-operator.md | 9 ++++ dev/runbooks/2-armada.md | 8 ++++ dev/runbooks/3-armadactl.md | 12 +++++ dev/runbooks/4-queue.md | 9 ++++ dev/runbooks/5-job.md | 23 ++++++++++ dev/runbooks/6-lookout.md | 9 ++++ dev/runbooks/7-join-cluster.md | 23 ++++++++++ 13 files changed, 250 insertions(+), 54 deletions(-) create mode 100644 dev/quickstart/job.yaml create mode 100644 dev/quickstart/minimal-setup.yaml create mode 100644 dev/runbooks/0-dependencies.md create mode 100644 dev/runbooks/1-operator.md create mode 100644 dev/runbooks/2-armada.md create mode 100644 dev/runbooks/3-armadactl.md create mode 100644 dev/runbooks/4-queue.md create mode 100644 dev/runbooks/5-job.md create mode 100644 dev/runbooks/6-lookout.md create mode 100644 dev/runbooks/7-join-cluster.md diff --git a/README.md b/README.md index 2b17d5d2..610a6f5e 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,20 @@ If you do not have a Kubernetes cluster to test against, you can start one using make create-dev-cluster ``` +### Install using Helm + +First, add the `gresearch` public Helm registry: +```bash +helm repo add gresearch https://g-research.github.io/charts +``` + +After that, you can install the Armada Operator using Helm: +```bash +helm install armada-operator gresearch/armada-operator --namespace armada-system --create-namespace +``` + +### Build & install from scratch + Run the following command to install all Armada external dependencies (Apache Pulsar, Redis, PostgreSQL, NGINX, Prometheus) ```bash make dev-setup @@ -61,42 +75,23 @@ make delete-dev-cluster This will totally destroy your development Kind cluster. -## Getting Started +### Local development -### Running on a Cluster - -1. Build and push your image to the location specified by `IMG`: - +Before running the Operator, we first need to install the CRDs by running the following command: ```bash -make docker-build docker-push IMG=/armada-operator:tag -``` - -2. Deploy the controller to the cluster with the image specified by `IMG`: - -```bash -make deploy IMG=/armada-operator:tag +make install ``` -3. Install Instances of Custom Resources: - +To run the operator locally, you can use the following command: ```bash -kubectl apply -f config/samples/ +make run ``` -### Uninstall CRDs -To delete the CRDs from the cluster: - +To uninstall the Operator CRDs, you can use the following command: ```bash make uninstall ``` -### Undeploy controller -UnDeploy the controller to the cluster: - -```bash -make undeploy -``` - ## Contributing Please feel free to contribute bug-reports or ideas for enhancements via @@ -106,7 +101,7 @@ Code contributions are also welcome. When submitting a pull-request please ensure it references a relevant issue as well as making sure all CI checks pass. -### Test All Changes +## Testing Please test contributions thoroughly before requesting reviews. At a minimum: ```bash @@ -123,33 +118,7 @@ are covered by automated tests and appear to be correct. This project aims to follow the Kubernetes [Operator pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) It uses [Controllers](https://kubernetes.io/docs/concepts/architecture/controller/) -which provides a reconcile function responsible for synchronizing resources untile the desired state is reached on the cluster - -### Test It Out -1. Install the CRDs into the cluster: - -```bash -make install -``` - -2. Run your controller (this will run in the foreground, so switch to a new terminal if you want to leave it running): - -```bash -make run -``` - -**NOTE:** You can also run this in one step by running: `make install run` - -### Modifying the API definitions -If you are editing the API definitions, generate the manifests such as CRs or CRDs using: - -```bash -make manifests -``` - -**NOTE:** Run `make --help` for more information on all potential `make` targets - -More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html) +which provides a reconcile function responsible for synchronizing resources untile the desired state is reached on the cluster ## License diff --git a/dev/quickstart/README.md b/dev/quickstart/README.md index bd5b0519..5605c2ab 100644 --- a/dev/quickstart/README.md +++ b/dev/quickstart/README.md @@ -111,3 +111,12 @@ Run the following commands to create the `armada-default` PriorityClass: ```bash kubectl apply -f dev/quickstart/priority-class.yaml ``` + +## Minimal setup + +Minimal setup for Armada which could only accept jobs would consist only of Armada Server and Executor deployed in the same cluster. + +Run the following command to install the minimal setup: +```bash +kubectl apply -f dev/quickstart/minimal-setup.yaml +``` diff --git a/dev/quickstart/armada-crds.yaml b/dev/quickstart/armada-crds.yaml index cb2cdde2..ebf5fa98 100644 --- a/dev/quickstart/armada-crds.yaml +++ b/dev/quickstart/armada-crds.yaml @@ -77,7 +77,7 @@ spec: # for the full list of configuration options. apiPort: 8080 corsAllowedOrigins: - - "*" + - "https://ui.demo.armadaproject.io" postgres: connection: host: postgres-postgresql.data.svc.cluster.local diff --git a/dev/quickstart/job.yaml b/dev/quickstart/job.yaml new file mode 100644 index 00000000..08aac302 --- /dev/null +++ b/dev/quickstart/job.yaml @@ -0,0 +1,22 @@ +queue: queue-kubecon-demo +jobSetId: job-set-1 +jobs: + - priority: 0 + podSpec: + terminationGracePeriodSeconds: 0 + restartPolicy: Never + containers: + - name: sleeper + image: alpine:latest + command: + - sh + args: + - -c + - sleep $(( (RANDOM % 60) + 10 )) + resources: + limits: + memory: 128Mi + cpu: 0.2 + requests: + memory: 128Mi + cpu: 0.2 diff --git a/dev/quickstart/minimal-setup.yaml b/dev/quickstart/minimal-setup.yaml new file mode 100644 index 00000000..18d13729 --- /dev/null +++ b/dev/quickstart/minimal-setup.yaml @@ -0,0 +1,57 @@ +apiVersion: install.armadaproject.io/v1alpha1 +kind: ArmadaServer +metadata: + name: armada-server + namespace: armada +spec: + pulsarInit: false + ingress: + ingressClass: "nginx" + clusterIssuer: "letsencrypt-dev" + hostNames: [] + replicas: 1 + image: + repository: gresearch/armada-server + tag: 0.3.92 + applicationConfig: + auth: + anonymousAuth: true + permissionGroupMapping: + submit_jobs: ["everyone"] + submit_any_jobs: ["everyone"] + create_queue: ["everyone"] + delete_queue: ["everyone"] + cancel_jobs: ["everyone"] + cancel_any_jobs: ["everyone"] + reprioritize_jobs: ["everyone"] + reprioritize_any_jobs: ["everyone"] + watch_events: ["everyone"] + watch_all_events: ["everyone"] + execute_jobs: ["everyone"] + postgres: + connection: + host: postgres-postgresql.data.svc.cluster.local + port: 5432 + user: postgres + password: psw + dbname: postgres + sslmode: disable + pulsar: + URL: pulsar://pulsar-broker.data.svc.cluster.local:6650 + redis: + addrs: + - redis-redis-ha.data.svc.cluster.local:6379 +--- +apiVersion: install.armadaproject.io/v1alpha1 +kind: Executor +metadata: + name: armada-executor + namespace: armada +spec: + image: + repository: gresearch/armada-executor + tag: 0.3.92 + applicationConfig: + apiConnection: + armadaUrl: armada-server.armada.svc.cluster.local:50051 + forceNoTls: true diff --git a/dev/runbooks/0-dependencies.md b/dev/runbooks/0-dependencies.md new file mode 100644 index 00000000..035d93c1 --- /dev/null +++ b/dev/runbooks/0-dependencies.md @@ -0,0 +1,46 @@ +# Dependencies + +Armada is a Kubernetes-native batch scheduler, and it requires a Kubernetes cluster for installation. +If you do not have a cluster, you can use [kind](https://kind.sigs.k8s.io/) to create a local cluster for testing purposes. +Let's run the following command to create a local cluster: +```bash +kind create cluster --name armada +``` + +Armada requires the following dependencies to be installed on the cluster: +* [PostgreSQL](https://www.postgresql.org/) - open source relational database +* [Redis](https://redis.io/) - open source, in-memory data store +* [Apache Pulsar](https://pulsar.apache.org/) - Cloud-Native, Distributed Messaging and Streaming +* [cert-manager](https://cert-manager.io/) - Kubernetes add-on to automate the management and issuance of TLS certificates from various issuing sources +* [NGINX Ingress Controller](https://kubernetes.github.io/ingress-nginx/) - Ingress controller that uses ConfigMap to store the NGINX configuration +* [Prometheus](https://prometheus.io/) - open-source systems monitoring and alerting toolkit +* OPTIONAL: [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) - add-on agent to generate and expose cluster-level metrics +* OPTIONAL: [Metrics Server](https://github.com/kubernetes-sigs/metrics-server) - add-on agent to collect resource metrics such as CPU and memory from nodes and pods + +Let's run the following commands to install the required dependencies: +```bash +kubectl create namespace data + +helm repo add jetstack https://charts.jetstack.io +helm upgrade --install \ + cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --version v1.13.1 \ + --set installCRDs=true + +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack --namespace monitoring --create-namespace + +helm repo add apache https://pulsar.apache.org/charts +helm upgrade --install pulsar apache/pulsar -f dev/quickstart/pulsar.values.yaml --namespace data + +helm repo add dandydev https://dandydeveloper.github.io/charts +helm upgrade --install redis dandydev/redis-ha -f dev/quickstart/redis.values.yaml --namespace data + +helm repo add bitnami https://charts.bitnami.com/bitnami +helm upgrade --install postgres bitnami/postgresql -f dev/quickstart/postgres.values.yaml --namespace data + +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm upgrade --install nginx ingress-nginx/ingress-nginx --namespace kube-system +``` diff --git a/dev/runbooks/1-operator.md b/dev/runbooks/1-operator.md new file mode 100644 index 00000000..b29a54e3 --- /dev/null +++ b/dev/runbooks/1-operator.md @@ -0,0 +1,9 @@ +# Armada Operator + +First, we need to install the Armada Operator, so we can install Armada by creating the Armada components CRDs. +Let's run the following commands to install the Armada Operator: + +```bash +helm repo add gresearch https://g-research.github.io/charts +helm install armada-operator gresearch/armada-operator --namespace armada-system --create-namespace +``` diff --git a/dev/runbooks/2-armada.md b/dev/runbooks/2-armada.md new file mode 100644 index 00000000..4ca808ab --- /dev/null +++ b/dev/runbooks/2-armada.md @@ -0,0 +1,8 @@ +# Armada + +Now we need to apply the Armada components CRDs and the Operator will install & configure the Armada cluster. +Let's run the following commands to create the Armada resources: +```bash +kubectl create namespace armada +kubectl apply -f dev/quickstart/armada-crds.yaml --namespace armada +``` diff --git a/dev/runbooks/3-armadactl.md b/dev/runbooks/3-armadactl.md new file mode 100644 index 00000000..9aa1206a --- /dev/null +++ b/dev/runbooks/3-armadactl.md @@ -0,0 +1,12 @@ +# Submit Job + +`armadactl` is a command line tool which is used to interact with the Armada API. +You can find a list of supported platforms and architectures in the Armada [Releases](https://github.com/armadaproject/armada/releases). +Let's run the following command to install the `armadactl` CLI: +```bash +curl -L -o /tmp/armadactl "https://github.com/armadaproject/armada/releases/download/v0.3.101/armadactl_0.3.101_darwin_all.tar.gz" && \ + tar -xzvf /tmp/armadactl && \ + mv armadactl /usr/local/bin/armadactl && \ + rm /tmp/armadactl && \ + chmod +x /usr/local/bin/armadactl +``` diff --git a/dev/runbooks/4-queue.md b/dev/runbooks/4-queue.md new file mode 100644 index 00000000..656dffe4 --- /dev/null +++ b/dev/runbooks/4-queue.md @@ -0,0 +1,9 @@ +# Armada Queues + +Armada queues are the mechanism to control access to the Armada cluster and ensure fair share. + +Let's run the following command to submit a job: + +```bash +armadactl create queue queue-kubecon-demo +``` diff --git a/dev/runbooks/5-job.md b/dev/runbooks/5-job.md new file mode 100644 index 00000000..127ad1c7 --- /dev/null +++ b/dev/runbooks/5-job.md @@ -0,0 +1,23 @@ +# Armada Jobs + +Armada Jobs are a superset of a Kubernetes PodSpec with additional Armada-specific fields. +```yaml +queue: queue-kubecon-demo +jobSetId: job-set-1 +jobs: + - priority: 0 + podSpec: + containers: + - name: sleeper + image: alpine:latest + command: + - sh + args: + - -c + - sleep $(( (RANDOM % 60) + 10 )) +``` + +Let's submit an Armada job in our Armada cluster by running the following command: +```bash +armadactl submit ./dev/quickstart/job.yaml +``` diff --git a/dev/runbooks/6-lookout.md b/dev/runbooks/6-lookout.md new file mode 100644 index 00000000..8a1e7c28 --- /dev/null +++ b/dev/runbooks/6-lookout.md @@ -0,0 +1,9 @@ +# Lookout + +Lookout is the Armada UI which shows information about jobs which are running in our Armada cluster. + +Let's run the following command to start Lookout: +```bash +kubectl port-forward svc/armada-lookout-v1 8080 --namespace armada +open http://localhost:8080 +``` diff --git a/dev/runbooks/7-join-cluster.md b/dev/runbooks/7-join-cluster.md new file mode 100644 index 00000000..a855ecd1 --- /dev/null +++ b/dev/runbooks/7-join-cluster.md @@ -0,0 +1,23 @@ +# Joining a new Kubernetes cluster to the Armada cluster + +Now that we have an Armada cluster running, let's join a new Kubernetes cluster to the Armada cluster. +We just need to install the Executor component in a Kubernetes cluster, and it will automagically join the Armada cluster. +```yaml +apiVersion: install.armadaproject.io/v1alpha1 +kind: Executor +metadata: + name: armada-executor + namespace: armada +spec: + image: + repository: gresearch/armada-executor + tag: 0.3.92 + applicationConfig: + apiConnection: + armadaUrl: server.demo.armadaproject.io:443 +``` + +Let's run the following command to install the Executor in our Kubernetes cluster: +```bash +kubectl apply -f dev/quickstart/executor.yaml +``` From cc3bf41ea7c529b1561b4f1e91ebdecac850bff0 Mon Sep 17 00:00:00 2001 From: Dejan Pejchev Date: Fri, 3 Nov 2023 14:58:52 +0100 Subject: [PATCH 2/3] remove references to kubecon in docs --- dev/quickstart/job.yaml | 2 +- dev/runbooks/0-dependencies.md | 2 +- dev/runbooks/4-queue.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/quickstart/job.yaml b/dev/quickstart/job.yaml index 08aac302..53384356 100644 --- a/dev/quickstart/job.yaml +++ b/dev/quickstart/job.yaml @@ -1,4 +1,4 @@ -queue: queue-kubecon-demo +queue: queue-a jobSetId: job-set-1 jobs: - priority: 0 diff --git a/dev/runbooks/0-dependencies.md b/dev/runbooks/0-dependencies.md index 035d93c1..5a4209ba 100644 --- a/dev/runbooks/0-dependencies.md +++ b/dev/runbooks/0-dependencies.md @@ -22,7 +22,7 @@ Let's run the following commands to install the required dependencies: kubectl create namespace data helm repo add jetstack https://charts.jetstack.io -helm upgrade --install \ +helm upgrade --install \ cert-manager jetstack/cert-manager \ --namespace cert-manager \ --create-namespace \ diff --git a/dev/runbooks/4-queue.md b/dev/runbooks/4-queue.md index 656dffe4..60cfb6b8 100644 --- a/dev/runbooks/4-queue.md +++ b/dev/runbooks/4-queue.md @@ -5,5 +5,5 @@ Armada queues are the mechanism to control access to the Armada cluster and ensu Let's run the following command to submit a job: ```bash -armadactl create queue queue-kubecon-demo +armadactl create queue queue-a ``` From 40fe75a30e66bee3284df757bee9ac403391ebd7 Mon Sep 17 00:00:00 2001 From: Dejan Pejchev Date: Mon, 4 Dec 2023 12:08:05 +0100 Subject: [PATCH 3/3] fix typo in README --- README.md | 2 +- dev/runbooks/5-job.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 610a6f5e..5f5dc10e 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ are covered by automated tests and appear to be correct. This project aims to follow the Kubernetes [Operator pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) It uses [Controllers](https://kubernetes.io/docs/concepts/architecture/controller/) -which provides a reconcile function responsible for synchronizing resources untile the desired state is reached on the cluster +which provides a reconcile function responsible for synchronizing resources until the desired state is reached on the cluster ## License diff --git a/dev/runbooks/5-job.md b/dev/runbooks/5-job.md index 127ad1c7..951c32a1 100644 --- a/dev/runbooks/5-job.md +++ b/dev/runbooks/5-job.md @@ -2,7 +2,7 @@ Armada Jobs are a superset of a Kubernetes PodSpec with additional Armada-specific fields. ```yaml -queue: queue-kubecon-demo +queue: queue-a jobSetId: job-set-1 jobs: - priority: 0