migrated tools install to a dedicated script that should work on work…

…stations

Makefile

@@ -163,17 +163,17 @@ aws-dev-down: build-installer
 		-e AWS_SESSION_TOKEN=$(shell aws configure get aws_session_token) \
 		substratus-installer aws-down.sh
 
-.PHONY: gcp-dev-run
+.PHONY: dev-run-gcp
 # Controller manager configuration #
-gcp-dev-run: export CLOUD=gcp
-gcp-dev-run: export GPU_TYPE=nvidia-l4
-gcp-dev-run: export PROJECT_ID=$(shell gcloud config get project)
-gcp-dev-run: export CLUSTER_NAME=substratus
-gcp-dev-run: export CLUSTER_LOCATION=us-central1
+dev-run-gcp: export CLOUD=gcp
+dev-run-gcp: export GPU_TYPE=nvidia-l4
+dev-run-gcp: export PROJECT_ID=$(shell gcloud config get project)
+dev-run-gcp: export CLUSTER_NAME=substratus
+dev-run-gcp: export CLUSTER_LOCATION=us-central1
 # Cloud manager configuration #
-gcp-dev-run: export GOOGLE_APPLICATION_CREDENTIALS=./secrets/gcp-manager-key.json
+dev-run-gcp: export GOOGLE_APPLICATION_CREDENTIALS=./secrets/gcp-manager-key.json
 # Run the controller manager and the cloud manager.
-gcp-dev-run: manifests kustomize install-crds
+dev-run-gcp: manifests kustomize install-crds
 	go run ./cmd/gcpmanager & \
 	go run ./cmd/controllermanager/main.go \
 		--sci-address=localhost:10080 \

docs/development.md

@@ -11,7 +11,7 @@ make gcp-dev-up
 Run Substratus control plane locally.
 
 ```sh
-make gcp-dev-run
+make dev-run-gcp
 ```
 
 Delete GCP infra.

install/Dockerfile

@@ -1,73 +1,11 @@
 FROM ubuntu:23.04
 
+ENV PATH $PATH:/workspace/scripts:/usr/local/gcloud/google-cloud-sdk/bin
 WORKDIR /workspace
-# Determine platform and architecture
-RUN ARCH=$(uname -m) && \
-  PLATFORM=$(uname -s | tr '[:upper:]' '[:lower:]') && \
-  if [ "$ARCH" = "aarch64" ]; then \
-  echo "AWSCLI_ARCH=aarch64" >> /etc/environment; \
-  echo "TERRAFORM_ARCH=arm64" >> /etc/environment; \
-  echo "PLATFORM_ARCH=${PLATFORM}_arm64" >> /etc/environment; \
-  elif [ "$ARCH" = "x86_64" ]; then \
-  echo "AWSCLI_ARCH=x86_64" >> /etc/environment; \
-  echo "TERRAFORM_ARCH=amd64" >> /etc/environment; \
-  echo "PLATFORM_ARCH=${PLATFORM}_amd64" >> /etc/environment; \
-  else \
-  echo "Unsupported architecture"; \
-  exit 1; \
-  fi
 
-# Source the environment file so that the variable is available in the current shell
-SHELL ["/bin/bash", "-c"]
-RUN source /etc/environment
-
-# Common
-RUN DEBIAN_FRONTEND="noninteractive" \
-  apt-get update && \
-  apt-get install -y \
-  gnupg \
-  software-properties-common \
-  unzip \
-  wget \
-  curl \
-  git \
-  gettext-base
-
-# AWS CLI
-RUN source /etc/environment && \
-  curl "https://awscli.amazonaws.com/awscli-exe-linux-${AWSCLI_ARCH}.zip" -o "awscliv2.zip" && \
-  unzip awscliv2.zip && \
-  ./aws/install
-
-# eksctl
-RUN source /etc/environment && \
-  curl -sLO "https://github.com/eksctl-io/eksctl/releases/latest/download/eksctl_${PLATFORM_ARCH}.tar.gz" && \
-  tar -xzf eksctl_${PLATFORM_ARCH}.tar.gz -C /tmp && rm eksctl_${PLATFORM_ARCH}.tar.gz && \
-  mv /tmp/eksctl /usr/local/bin
-
-# Terraform
-RUN source /etc/environment && \
-  wget https://releases.hashicorp.com/terraform/1.4.5/terraform_1.4.5_linux_${TERRAFORM_ARCH}.zip && \
-  unzip terraform_1.4.5_linux_${TERRAFORM_ARCH}.zip && \
-  mv terraform /usr/local/bin/ && \
-  terraform --version
-
-# Google Cloud (gcloud)
-RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz && \
-  mkdir -p /usr/local/gcloud \
-  && tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz \
-  && /usr/local/gcloud/google-cloud-sdk/install.sh
-ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin
-RUN gcloud --version
-RUN gcloud components install gke-gcloud-auth-plugin kubectl
+COPY scripts scripts
 
-# Helm
-RUN curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
-RUN chmod 700 /tmp/get_helm.sh
-RUN /tmp/get_helm.sh
+RUN scripts/get-tools.sh
 
-# Local files
 COPY terraform terraform
 COPY kubernetes kubernetes
-COPY scripts scripts
-ENV PATH $PATH:/workspace/scripts

install/scripts/aws-down.sh

@@ -24,13 +24,14 @@ export ARTIFACTS_BUCKET_NAME=${AWS_ACCOUNT_ID}-${CLUSTER_NAME}-artifacts
 envsubst <${KUBERENTES_DIR}/aws/eks-cluster.yaml.tpl >${KUBERENTES_DIR}/aws/eks-cluster.yaml
 eksctl delete cluster -f ${KUBERENTES_DIR}/aws/eks-cluster.yaml || true
 
-aws sqs delete-queue \
-  --queue-url https://sqs.${REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/substratus \
-  --region ${REGION} || true
 aws cloudformation delete-stack \
   --stack-name "Karpenter-${CLUSTER_NAME}" \
   --region ${REGION} || true
 
+aws sqs delete-queue \
+  --queue-url https://sqs.${REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/substratus \
+  --region ${REGION} || true
+
 aws ecr delete-repository \
   --repository-name ${ARTIFACTS_REPO_NAME} \
   --region ${REGION} >/dev/null || true

install/scripts/get-tools.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+terraform_version="1.4.5"
+
+tempout=$(mktemp -d)
+
+# Determine platform and architecture
+arch=$(uname -m)
+platform=$(uname -s | tr '[:upper:]' '[:lower:]')
+
+if [[ "$arch" = "aarch64" || "$arch" = "arm64" ]]; then
+  awscli_arch=aarch64
+  terraform_arch=arm64
+  platform_arch=${platform}_arm64
+elif [ "$arch" = "x86_64" ]; then
+  awscli_arch=x86_64
+  terraform_arch=amd64
+  platform_arch=${platform}_amd64
+else
+  echo "Unsupported architecture"
+  exit 1
+fi
+
+# install all our common tools
+if [ "${platform}" == "linux" ]; then
+  DEBIAN_FRONTEND="noninteractive" \
+    apt-get update
+  apt-get install -y \
+    gnupg \
+    software-properties-common \
+    unzip \
+    curl \
+    git \
+    python3-venv \
+    gettext-base
+elif [ "${platform}" == "darwin" ]; then
+  brew install \
+    gnupg \
+    unzip \
+    curl \
+    git \
+    gettext
+else
+  echo "Unsupported platform"
+  exit 1
+fi
+
+install_awscli() {
+  curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "${tempout}/awscli-bundle.zip"
+  unzip "${tempout}/awscli-bundle.zip" -d ${tempout}
+  python3 "${tempout}/awscli-bundle/install" -i /usr/local/aws -b /usr/local/bin/aws
+}
+
+install_eksctl() {
+  curl -sL "https://github.com/eksctl-io/eksctl/releases/latest/download/eksctl_${platform_arch}.tar.gz" \
+    -o ${tempout}/eksctl.tar.gz
+  tar -xzf ${tempout}/eksctl.tar.gz -C /tmp
+  (mv /tmp/eksctl /usr/local/bin || sudo mv /tmp/eksctl /usr/local/bin)
+}
+
+install_terraform() {
+  curl https://releases.hashicorp.com/terraform/${terraform_version}/terraform_${terraform_version}_${platform}_${terraform_arch}.zip \
+    -o ${tempout}/terraform.zip
+  unzip ${tempout}/terraform.zip -d ${tempout}
+  (mv ${tempout}/terraform /usr/local/bin/ || sudo mv ${tempout}/terraform /usr/local/bin/)
+}
+
+install_gcloud() {
+  curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz \
+    -o ${tempout}/google-cloud-sdk.tar.gz
+  mkdir -p /usr/local/gcloud
+  tar -C /usr/local/gcloud -xvf ${tempout}/google-cloud-sdk.tar.gz
+  /usr/local/gcloud/google-cloud-sdk/install.sh
+  gcloud components install gke-gcloud-auth-plugin kubectl
+}
+
+install_helm() {
+  curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 \
+    -o ${tempout}/get_helm.sh
+  chmod 700 ${tempout}/get_helm.sh
+  ${tempout}/get_helm.sh
+}
+
+if ! command -v aws &>/dev/null; then install_awscli; fi
+if ! command -v eksctl &>/dev/null; then install_eksctl; fi
+if ! command -v terraform &>/dev/null; then install_terraform; fi
+if ! command -v gcloud &>/dev/null; then install_gcloud; fi
+if ! command -v kubectl &>/dev/null; then install_gcloud; fi
+if ! command -v helm &>/dev/null; then install_helm; fi
+
+rm -r ${tempout}
+
+echo "Installation complete!"