From 7c000ec1bcd9df4bd86b500d5fb23e49edb01a0b Mon Sep 17 00:00:00 2001 From: Aryeh Harris Date: Wed, 23 Feb 2022 18:51:09 -0500 Subject: [PATCH 1/2] Remove /pulumi --- pulumi/Pulumi.yaml | 3 - pulumi/README.md | 136 ----- pulumi/fluent-bit.yaml | 362 ------------ pulumi/index.ts | 179 ------ pulumi/nginx_ingress_values.yaml | 16 - pulumi/package.json | 13 - pulumi/tsconfig.json | 19 - pulumi/yarn.lock | 956 ------------------------------- 8 files changed, 1684 deletions(-) delete mode 100644 pulumi/Pulumi.yaml delete mode 100644 pulumi/README.md delete mode 100644 pulumi/fluent-bit.yaml delete mode 100644 pulumi/index.ts delete mode 100644 pulumi/nginx_ingress_values.yaml delete mode 100644 pulumi/package.json delete mode 100644 pulumi/tsconfig.json delete mode 100644 pulumi/yarn.lock diff --git a/pulumi/Pulumi.yaml b/pulumi/Pulumi.yaml deleted file mode 100644 index a8a57441c..000000000 --- a/pulumi/Pulumi.yaml +++ /dev/null @@ -1,3 +0,0 @@ -name: tezos-k8s -runtime: nodejs -description: A minimal AWS TypeScript Pulumi program diff --git a/pulumi/README.md b/pulumi/README.md deleted file mode 100644 index 35c0b2091..000000000 --- a/pulumi/README.md +++ /dev/null @@ -1,136 +0,0 @@ -# Using Pulumi to deploy a private chain in AWS - -This README will walk you through setting up a Tezos based private -blockchain where you will spin up many bootstrap nodes as well as additional -peer nodes if you'd like. We will demonstrate this in AWS EKS (Elastic -Kubernetes Service). - -This guide describes how to deploy the cluster as a developer of -this framework. That is, this is the way to update the framework -that deploys said networks. - -We are not using Zerotier in this example. - -## Prerequisites - -- python3 - - pip - - python3-venv -- [docker](https://docs.docker.com/get-docker/) - - make sure that you add your username to the docker group - - https://docs.docker.com/engine/install/linux-postinstall/ -- [kubectl](https://kubernetes.io/docs/reference/kubectl/kubectl/) -- [helm](https://helm.sh/) -- AWS CLI -- nodejs -- pulumi - - install pulumi - - create and account and tokens - - pulumi has some modules to install - -## mkchain - -mkchain is a python script that generates Helm values, which Helm then -uses to create your Tezos chain on k8s. - -Follow _just_ the [Install mkchain](./mkchain/README.md#install-mkchain) -step in `./mkchain/README.md`. See there for more info on how you can -customize your chain. At the moment, this is development code and so -you can't use the mkchain that pip installs by default. You must rather -do: - -```shell -python3 -m venv .venv -. .venv/bin/activate -pip install wheel && pip install ./mkchain -``` - -at the top level of this git repository. Note the ./mkchain, this tells -pip to install the version that you have currently checked out. - -Set as an environment variable the name you would like to give to your chain: - -```shell -export CHAIN_NAME=pulumi -``` - -We'll assume that you've set that from this point in. - -## Helm dependencies - -You need to run: - -```shell -helm dependency update charts/tezos -``` - -before we begin and after certain changes to the helm charts. - -## Create your chain values files: - -Run the following commands to create the helm chart values files. - -```shell -mkchain --number-of-bakers 10 $CHAIN_NAME -``` - -## Use Pulumi to "make it so" - -First you must set up a Pulumi "stack". You can define multiple -stacks and switch between them for deploying different sets of nodes. - -```shell -pulumi stack init my_name -``` - -We look for the values file as `${STACK_NAME}_values.yaml`. - -### Configure Pulumi - -Pulumi has the ability to set configuration parameters for each stack -that it maintains. Some of the parameters are generic and some are -specific to tezos-k8s. It is required to set an AWS region, when -deploying to AWS: - -```shell -pulumi config set aws:region us-east-2 -``` - -We also defined a number of parameters for tezos-k8s: - -```shell -pulumi config set max-cluster-capacity 100 -pulumi config set nodes-per-vm 8 -pulumi config set cloudwatch true -pulumi config set rpc-auth true -``` - -## Actually "make it so" this time - -```shell -pulumi up -``` - -The last command will take quite a long time. Don't kill it, that -can leave pulumi in a bad state. - -## Examine the network that you are creating. - -Pulumi will output a `kubeconfig.json` that you can use with your -existing tools to examine the cluster you just created. You can -have a look at the bakers and nodes via: - -```shell -pulumi stack output kubeconfig > /tmp/kubeconfig.json -export KUBECONFIG=/tmp/kubeconfig.json -kubectl -n tezos get pods -``` - -As long as that works, you should be able to examine the cluster -using `kubectl`, `k9s`, etc. - -If things don't come up, or if the behaviour is odd, try: - -```shell -kubectl -n tezos describe statefulsets -``` diff --git a/pulumi/fluent-bit.yaml b/pulumi/fluent-bit.yaml deleted file mode 100644 index 5170c6509..000000000 --- a/pulumi/fluent-bit.yaml +++ /dev/null @@ -1,362 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: fluent-bit - namespace: amazon-cloudwatch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: fluent-bit-role -rules: - - nonResourceURLs: - - /metrics - verbs: - - get - - apiGroups: [""] - resources: - - namespaces - - pods - - pods/logs - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: fluent-bit-role-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: fluent-bit-role -subjects: - - kind: ServiceAccount - name: fluent-bit - namespace: amazon-cloudwatch ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: fluent-bit-config - namespace: amazon-cloudwatch - labels: - k8s-app: fluent-bit -data: - fluent-bit.conf: | - [SERVICE] - Flush 5 - Log_Level info - Daemon off - Parsers_File parsers.conf - HTTP_Server ${HTTP_SERVER} - HTTP_Listen 0.0.0.0 - HTTP_Port ${HTTP_PORT} - storage.path /var/fluent-bit/state/flb-storage/ - storage.sync normal - storage.checksum off - storage.backlog.mem_limit 5M - - @INCLUDE application-log.conf - @INCLUDE dataplane-log.conf - @INCLUDE host-log.conf - - application-log.conf: | - [INPUT] - Name tail - Tag application.* - Exclude_Path /var/log/containers/cloudwatch-agent*, /var/log/containers/fluent-bit*, /var/log/containers/aws-node*, /var/log/containers/kube-proxy* - Path /var/log/containers/*.log - Docker_Mode On - Docker_Mode_Flush 5 - Docker_Mode_Parser container_firstline - Parser docker - DB /var/fluent-bit/state/flb_container.db - Mem_Buf_Limit 50MB - Skip_Long_Lines On - Refresh_Interval 10 - Rotate_Wait 30 - storage.type filesystem - Read_from_Head ${READ_FROM_HEAD} - - [INPUT] - Name tail - Tag application.* - Path /var/log/containers/fluent-bit* - Parser docker - DB /var/fluent-bit/state/flb_log.db - Mem_Buf_Limit 5MB - Skip_Long_Lines On - Refresh_Interval 10 - Read_from_Head ${READ_FROM_HEAD} - - [INPUT] - Name tail - Tag application.* - Path /var/log/containers/cloudwatch-agent* - Docker_Mode On - Docker_Mode_Flush 5 - Docker_Mode_Parser cwagent_firstline - Parser docker - DB /var/fluent-bit/state/flb_cwagent.db - Mem_Buf_Limit 5MB - Skip_Long_Lines On - Refresh_Interval 10 - Read_from_Head ${READ_FROM_HEAD} - - [FILTER] - Name kubernetes - Match application.* - Kube_URL https://kubernetes.default.svc:443 - Kube_Tag_Prefix application.var.log.containers. - Merge_Log On - Merge_Log_Key log_processed - K8S-Logging.Parser On - K8S-Logging.Exclude Off - Labels Off - Annotations Off - - [OUTPUT] - Name cloudwatch_logs - Match application.* - region ${AWS_REGION} - log_group_name /aws/containerinsights/${CLUSTER_NAME}/application - log_stream_prefix ${HOST_NAME}- - auto_create_group true - extra_user_agent container-insights - - dataplane-log.conf: | - [INPUT] - Name systemd - Tag dataplane.systemd.* - Systemd_Filter _SYSTEMD_UNIT=docker.service - DB /var/fluent-bit/state/systemd.db - Path /var/log/journal - Read_From_Tail ${READ_FROM_TAIL} - - [INPUT] - Name tail - Tag dataplane.tail.* - Path /var/log/containers/aws-node*, /var/log/containers/kube-proxy* - Docker_Mode On - Docker_Mode_Flush 5 - Docker_Mode_Parser container_firstline - Parser docker - DB /var/fluent-bit/state/flb_dataplane_tail.db - Mem_Buf_Limit 50MB - Skip_Long_Lines On - Refresh_Interval 10 - Rotate_Wait 30 - storage.type filesystem - Read_from_Head ${READ_FROM_HEAD} - - [FILTER] - Name modify - Match dataplane.systemd.* - Rename _HOSTNAME hostname - Rename _SYSTEMD_UNIT systemd_unit - Rename MESSAGE message - Remove_regex ^((?!hostname|systemd_unit|message).)*$ - - [FILTER] - Name aws - Match dataplane.* - imds_version v1 - - [OUTPUT] - Name cloudwatch_logs - Match dataplane.* - region ${AWS_REGION} - log_group_name /aws/containerinsights/${CLUSTER_NAME}/dataplane - log_stream_prefix ${HOST_NAME}- - auto_create_group true - extra_user_agent container-insights - - host-log.conf: | - [INPUT] - Name tail - Tag host.dmesg - Path /var/log/dmesg - Parser syslog - DB /var/fluent-bit/state/flb_dmesg.db - Mem_Buf_Limit 5MB - Skip_Long_Lines On - Refresh_Interval 10 - Read_from_Head ${READ_FROM_HEAD} - - [INPUT] - Name tail - Tag host.messages - Path /var/log/messages - Parser syslog - DB /var/fluent-bit/state/flb_messages.db - Mem_Buf_Limit 5MB - Skip_Long_Lines On - Refresh_Interval 10 - Read_from_Head ${READ_FROM_HEAD} - - [INPUT] - Name tail - Tag host.secure - Path /var/log/secure - Parser syslog - DB /var/fluent-bit/state/flb_secure.db - Mem_Buf_Limit 5MB - Skip_Long_Lines On - Refresh_Interval 10 - Read_from_Head ${READ_FROM_HEAD} - - [FILTER] - Name aws - Match host.* - imds_version v1 - - [OUTPUT] - Name cloudwatch_logs - Match host.* - region ${AWS_REGION} - log_group_name /aws/containerinsights/${CLUSTER_NAME}/host - log_stream_prefix ${HOST_NAME}. - auto_create_group true - extra_user_agent container-insights - - parsers.conf: | - [PARSER] - Name docker - Format json - Time_Key time - Time_Format %Y-%m-%dT%H:%M:%S.%LZ - - [PARSER] - Name syslog - Format regex - Regex ^(?