From 1d2f9f3613b1f0ee5933838b11d0ccfa5590e72d Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 4 Jun 2020 16:08:30 -0400 Subject: [PATCH 1/4] Make markdown files be one sentence per line --- README.md | 13 +++++++------ docs/cli/README.md | 6 ++++-- docs/cli/template.md | 11 ++++++----- docs/cli/workflow.md | 7 ++++--- docs/components.md | 26 ++++++++++++++++++-------- docs/concepts.md | 35 ++++++++++++++++++++++++++--------- docs/hello-world.md | 9 ++++++--- docs/setup.md | 9 ++++++--- docs/troubleshoot.md | 10 ++++++---- docs/writing-workflow.md | 18 +++++++++++------- 10 files changed, 94 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index e68fb4044..ca235f393 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,14 @@ -# Tinkerbell +# Tinkerbell [![Build Status](https://cloud.drone.io/api/badges/tinkerbell/tink/status.svg)](https://cloud.drone.io/tinkerbell/tink) -Tinkerbell is a bare metal provisioning and workflow engine comprised of three major components: a DHCP server ([boots](https://github.com/packethost/boots)), a workflow engine (tinkerbell, this repository), and a metadata service ([hegel](https://github.com/packethost/hegel)). The workflow engine is comprised of a server and a CLI, which communicate over gRPC. The CLI is used to create a workflow along with its building blocks, templates and targeted hardware. +Tinkerbell is a bare metal provisioning and workflow engine comprised of three major components: a DHCP server ([boots](https://github.com/packethost/boots)), a workflow engine (tinkerbell, this repository), and a metadata service ([hegel](https://github.com/packethost/hegel)). +The workflow engine is comprised of a server and a CLI, which communicate over gRPC. +The CLI is used to create a workflow along with its building blocks, templates and targeted hardware. -# Packet Workflow +# Packet Workflow -A Packet Workflow is an open-source microservice that’s responsible for handling flexible, bare metal -provisioning workflows, that is... +A Packet Workflow is an open-source microservice that’s responsible for handling flexible, bare metal provisioning workflows, that is... - standalone and does not need the Packet API to function - contains `Boots`, `Tinkerbell`, `Osie`, and workers - can bootstrap any remote worker using `Boots + Osie` @@ -15,7 +16,7 @@ provisioning workflows, that is... - receive, manipulate, and save runtime data ## Content - + - [Setup](docs/setup.md) - [Components](docs/components.md) - [Boots](docs/components.md#boots) diff --git a/docs/cli/README.md b/docs/cli/README.md index 056bfd9f2..1cac870ad 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -6,7 +6,9 @@ Command line interface for Packet Workflow. ### Synopsis -Command line interface for Packet Workflow. The CLI allows you to update the hardware details with respect to a worker machine. It also enables you to create a template which is eventually used to create a workflow. +Command line interface for Packet Workflow. +The CLI allows you to update the hardware details with respect to a worker machine. +It also enables you to create a template which is eventually used to create a workflow. ### Operations @@ -25,7 +27,7 @@ Command line interface for Packet Workflow. The CLI allows you to update the har ### See Also - - [tink hardware](hardware.md) - Hardware (worker) data operations + - [tink hardware](hardware.md) - Hardware (worker) data operations - [tink template](template.md) - Template operations - [tink workflow](workflow.md) - Workflow operations diff --git a/docs/cli/template.md b/docs/cli/template.md index 232de4578..044ce57e1 100644 --- a/docs/cli/template.md +++ b/docs/cli/template.md @@ -6,7 +6,7 @@ Template operations. Template operations: ```shell - create create a workflow template + create create a workflow template delete delete a template get get a template list list all saved templates @@ -21,13 +21,14 @@ Template operations: ### Examples - - The following command creates a workflow template using the `sample.tmpl` file and save it as `sample`. It returns a UUID for the newly created template. + - The following command creates a workflow template using the `sample.tmpl` file and save it as `sample`. + It returns a UUID for the newly created template. ```shell $ tink template create -n -p $ tink template create -n sample -p /tmp/sample.tmpl - ``` + ``` - - List all the templates + - List all the templates ```shell $ tink template list ``` @@ -46,6 +47,6 @@ Template operations: ### See Also - - [tink hardware](hardware.md) - Hardware (worker) data operations + - [tink hardware](hardware.md) - Hardware (worker) data operations - [tink workflow](workflow.md) - Workflow operations diff --git a/docs/cli/workflow.md b/docs/cli/workflow.md index 40220dc73..b6f955837 100644 --- a/docs/cli/workflow.md +++ b/docs/cli/workflow.md @@ -28,12 +28,13 @@ Workflow operations: $ tink workflow create -t -r $ tink workflow create -t edb80a56-b1f2-4502-abf9-17326324192b -r {"device_1": "mac/IP"} ``` - #### Note: - 1. The key used in the above command which is "device_1" should be in sync with "worker" field in the template. Click [here](../concepts.md) to check the template structure. + #### Note: + 1. The key used in the above command which is "device_1" should be in sync with "worker" field in the template. + Click [here](../concepts.md) to check the template structure. 2. These keys can only contain letter, numbers and underscore. ### See Also - [tink hardware](hardware.md) - Hardware (worker) data operations - [tink template](template.md) - Template operations - + diff --git a/docs/components.md b/docs/components.md index 3dfd8f279..3359f20f7 100644 --- a/docs/components.md +++ b/docs/components.md @@ -2,7 +2,9 @@ ### Boots -Handles DHCP requests, hands out IPs, and serves up iPXE. It also uses the Tinkerbell client to pull and push hardware data. `boots` will only respond to a predefined set of MAC addresses so it can be deployed in an existing network without interfering with existing DHCP infrastructure. +Handles DHCP requests, hands out IPs, and serves up iPXE. +It also uses the Tinkerbell client to pull and push hardware data. +`boots` will only respond to a predefined set of MAC addresses so it can be deployed in an existing network without interfering with existing DHCP infrastructure. ### Osie @@ -10,29 +12,37 @@ Installs operating systems and handles deprovisioning. ### Tinkerbell -Service responsible for processing workflows. It is comprised of a server and a CLI, which communicate over gRPC. The CLI is used to create a workflow along with its building blocks, i.e., a template hardware devices. +Service responsible for processing workflows. +It is comprised of a server and a CLI, which communicate over gRPC. +The CLI is used to create a workflow along with its building blocks, i.e., a template hardware devices. ### Hegel -Metadata service used by Tinkerbell and Osie during provisioning. It collects data from Tinkerbell and transforms it into a JSON format to be consumed as metadata. +Metadata service used by Tinkerbell and Osie during provisioning. +It collects data from Tinkerbell and transforms it into a JSON format to be consumed as metadata. ### Database -We use [PostgreSQL](https://www.postgresql.org/), also known as Postgres, as our data store. Postgres is a free and open-source relational database management system that emphasizes extensibility and technical standards compliance. It is designed to handle a range of workloads, from single machines to data warehouses or Web services with many concurrent users. +We use [PostgreSQL](https://www.postgresql.org/), also known as Postgres, as our data store. +Postgres is a free and open-source relational database management system that emphasizes extensibility and technical standards compliance. +It is designed to handle a range of workloads, from single machines to data warehouses or Web services with many concurrent users. ### Image Repository -Depending on your use case, you can choose to use [Quay](https://quay.io/) or [DockerHub](https://hub.docker.com/) as the registry to store component images. You can use the same registry to store all of the action images used for a workflow. +Depending on your use case, you can choose to use [Quay](https://quay.io/) or [DockerHub](https://hub.docker.com/) as the registry to store component images. +You can use the same registry to store all of the action images used for a workflow. -On the other hand, if you want to keep things local, you can also setup a secure private Docker registry to hold all your images locally. +On the other hand, if you want to keep things local, you can also setup a secure private Docker registry to hold all your images locally. ### Fluent Bit -[Fluent Bit](https://fluentbit.io/) is an open source and multi-platform Log Processor and Forwarder which allows you to collect data/logs from different sources, unify and send them to multiple destinations. The components write their logs to `stdout` and these logs are then collected by Fluent Bit and pushed to Elasticsearch. +[Fluent Bit](https://fluentbit.io/) is an open source and multi-platform Log Processor and Forwarder which allows you to collect data/logs from different sources, unify and send them to multiple destinations. +The components write their logs to `stdout` and these logs are then collected by Fluent Bit and pushed to Elasticsearch. ### Elasticsearch -[Elasticsearch](https://www.elastic.co/) is a distributed, open source search and analytics engine for all types of data, including textual, numerical, geospatial, structured, and unstructured. Fluent Bit collects the logs from each component and pushes them into Elasticsearch for storage and analysis purposes. +[Elasticsearch](https://www.elastic.co/) is a distributed, open source search and analytics engine for all types of data, including textual, numerical, geospatial, structured, and unstructured. +Fluent Bit collects the logs from each component and pushes them into Elasticsearch for storage and analysis purposes. ### Kibana diff --git a/docs/concepts.md b/docs/concepts.md index 7bb45e83a..b219bba1d 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -5,9 +5,14 @@ A *hardware device* is defined separately and is substituted in a template at th ### Template -A template is a Go template based definition that defines the overall flow of a workflow. A user must write a template based on a valid template format. Template can consist of custom variable which can be substituted before execution. For example, a hardware device is defined separately and is substituted in a template at the time of creating a workflow. +A template is a Go template based definition that defines the overall flow of a workflow. +A user must write a template based on a valid template format. +Template can consist of custom variable which can be substituted before execution. +For example, a hardware device is defined separately and is substituted in a template at the time of creating a workflow. -A template is stored as a blob in the database and is parsed later during the creation of a worflow. A user can CRUD a template using the CLI (`tink template`). Here is a sample workflow template: +A template is stored as a blob in the database and is parsed later during the creation of a worflow. +A user can CRUD a template using the CLI (`tink template`). +Here is a sample workflow template: ```yaml version: '0.1' @@ -43,17 +48,24 @@ tasks: - /statedir:/statedir ``` -A template comprises Tasks, which are executed in a sequential manner. A task can consists multiple Actions. As can be in the above example, a task supports volumes and environment variables. The volumes and environment variables defined for a particular task level are inherited by each action in that particular task. +A template comprises Tasks, which are executed in a sequential manner. +A task can consists multiple Actions. +As can be in the above example, a task supports volumes and environment variables. +The volumes and environment variables defined for a particular task level are inherited by each action in that particular task. -It is important to note that an action can also have its own volumes and environment variables. Therefore, any entry at an action will overwrite the value defined at the task level. For example, in the above template the `MIRROR_HOST` environment variable defined at action `disk-partition` will overwrite the value defined at task level. However, the other actions will receive the original value defined at the task level. +It is important to note that an action can also have its own volumes and environment variables. +Therefore, any entry at an action will overwrite the value defined at the task level. +For example, in the above template the `MIRROR_HOST` environment variable defined at action `disk-partition` will overwrite the value defined at task level. +However, the other actions will receive the original value defined at the task level. ### Provisioner -The provisioner machine is the main driver for executing a workflow. A provisioner houses the following components: +The provisioner machine is the main driver for executing a workflow. +A provisioner houses the following components: - Database (Postgres) - Tinkerbell (CLI and server) - - Boots + - Boots - Hegel - Image Registry (optional) - Elasticsearch @@ -65,14 +77,19 @@ It is up to you if you would like to divide these components into multiple serve ### Worker -Any node that has its data being pushed into Tinkerbell can become a part of a workflow. A worker can be a part of multiple workflows. +Any node that has its data being pushed into Tinkerbell can become a part of a workflow. +A worker can be a part of multiple workflows. -When the node boots, a worker container starts and connects with provisioner to check if there is any task (may be from different workflows) that it can execute. After the completion of an action, the worker sends action status to provisioner. When all workflows which are related to a worker are complete, a worker can terminate. +When the node boots, a worker container starts and connects with provisioner to check if there is any task (may be from different workflows) that it can execute. +After the completion of an action, the worker sends action status to provisioner. +When all workflows which are related to a worker are complete, a worker can terminate. ### Ephemeral Data -The workers that are part of a workflow might require to share some data. This can take the form of a light JSON like below, or some binary files that other workers might require to complete their action. For instance, a worker may add the following data: +The workers that are part of a workflow might require to share some data. +This can take the form of a light JSON like below, or some binary files that other workers might require to complete their action. +For instance, a worker may add the following data: ```json {"operating_system": "ubuntu_18_04", "mac_addr": "F5:C9:E2:99:BD:9B", "instance_id": "123e4567-e89b-12d3-a456-426655440000"} diff --git a/docs/hello-world.md b/docs/hello-world.md index 7c7889b62..a5b0c5970 100644 --- a/docs/hello-world.md +++ b/docs/hello-world.md @@ -4,7 +4,8 @@ Here is an example to execute the most simple workflow that says "Hello World!". ### Prerequisite -You have a setup ready with a provisioner and a worker node. If not, please follow the steps [here](setup.md) to complete the setup. +You have a setup ready with a provisioner and a worker node. +If not, please follow the steps [here](setup.md) to complete the setup. ### Hardware Data @@ -45,7 +46,8 @@ A few details to note: - `ip_addresses.address` is the IP to hand out - `network_ports` is the worker MAC address -Now that we have the hardware data, we need to push it into the database. In order to do so, remove the extra spaces in the above JSON and use the following command to push the data: +Now that we have the hardware data, we need to push it into the database. +In order to do so, remove the extra spaces in the above JSON and use the following command to push the data: ``` $ tink hardware push '' ``` @@ -57,7 +59,8 @@ $ tink hardware mac ### Action images -The workflow will have a single task that will have a single action. The action here is to say `Hello-world!`, so we will push the action image to the registry running on the provisioner: +The workflow will have a single task that will have a single action. +The action here is to say `Hello-world!`, so we will push the action image to the registry running on the provisioner: ```shell $ docker pull hello-world $ docker tag hello-world /hello-world diff --git a/docs/setup.md b/docs/setup.md index c1e2428aa..d7a07249a 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -12,7 +12,8 @@ $ ./setup.sh ``` ### Declarative Mode -You can also execute the setup in declarative mode. In order to do so, define the following environment variables (examples here): +You can also execute the setup in declarative mode. +In order to do so, define the following environment variables (examples here): ```shell export TB_INTERFACE=network-interface # enp1s0f1 export TB_NETWORK=network-with-cidr # 192.168.1.0/29 @@ -28,7 +29,8 @@ $ curl https://raw.githubusercontent.com/tinkerbell/tink/master/setup.sh | bash ### Good to know - All the environment variables are kept in the `envrc` file, which is generated from the setup itself. - It is advised that you keep all the environment variables in the same file. - - It is important to note that if you execute the setup the again, a new `envrc` will be generated. However, the existing environment configuration be saved as `envrc.bak`. + - It is important to note that if you execute the setup the again, a new `envrc` will be generated. + However, the existing environment configuration be saved as `envrc.bak`. - The setup removes all the `.tar.gz` files downloaded in process. ### For Packet Environment @@ -51,5 +53,6 @@ The script was tested with: ### What's Next -Once you have the provisioner setup successfully, you can try executing your first workflow. Follow the steps described in [here](hello-world.md) to say "Hello World!" with a workflow. +Once you have the provisioner setup successfully, you can try executing your first workflow. +Follow the steps described in [here](hello-world.md) to say "Hello World!" with a workflow. diff --git a/docs/troubleshoot.md b/docs/troubleshoot.md index 0847ec8b3..b6e346dc6 100644 --- a/docs/troubleshoot.md +++ b/docs/troubleshoot.md @@ -9,9 +9,10 @@ A system restart is required. The system detected an exception during the UEFI pre-boot environment. ``` -Enable UEFI boot from the BIOS settings. To enable PXE boot, execute the following commands: +Enable UEFI boot from the BIOS settings. +To enable PXE boot, execute the following commands: ```shell -$ apk add ipmitool +$ apk add ipmitool $ ipmitool chassis bootdev pxe ``` @@ -26,9 +27,10 @@ No boot device available or Operating System detected. Please ensure a compatible bootable media is available. ``` -Ensure that you have pushed the hardware data to Tinkerbell having the correct MAC address. You can push the data with `tink push `. +Ensure that you have pushed the hardware data to Tinkerbell having the correct MAC address. +You can push the data with `tink push `. -### Can't login to private registry +### Can't login to private registry ```shell Login did not succeed, error: Error response from daemon: Get https://192.168.1.1/v2/: x509: certificate signed by unknown authority (possibly because of "crypto/rsa: verification error" while trying to verify candidate authority certificate "Autogenerated CA") diff --git a/docs/writing-workflow.md b/docs/writing-workflow.md index 718f2478f..38bee8ced 100644 --- a/docs/writing-workflow.md +++ b/docs/writing-workflow.md @@ -7,7 +7,7 @@ Any workflow comprises two building blocks: worker (targeted hardware) and a tem A worker is targeted hardware on which workflow needs to run. User need to push the hardware details as per the below command: ```shell - $ tink hardware push "" + $ tink hardware push "" ``` ### Creating a [template](concepts.md#template) @@ -46,10 +46,11 @@ tasks: Key points: - `global_timeout` is in seconds. - Any worflow that exceeds the global timeout will be terminated and marked as failed. - - Each action has its own timeout. If an action reaches its timeout, it is marked as failed and so is the workflow. + - Each action has its own timeout. + If an action reaches its timeout, it is marked as failed and so is the workflow. - An action cannot have space (` `) in its name. - Environment variables and volumes at action level overwrites the values for duplicate keys defined at task level. - + A worker can be accessed in a template like: ``` @@ -60,20 +61,23 @@ A worker can be accessed in a template like: The following command creates a workflow template and returns a UUID: ```shell $ tink template create -n sample -p /tmp/sample.tmpl -``` +``` ### Creating a [workflow](concepts.md#workflow) -We can create a workflow using the above created (or existing) template and worker. +We can create a workflow using the above created (or existing) template and worker. ```shell $ tink workflow create -t -r '{worker machines in json format}' $ tink workflow create -t edb80a56-b1f2-4502-abf9-17326324192b -r '{"device_1":"mac/IP"}' ``` -The above command returns a UUID for the workflow thus created. The workflow ID can be used for getting further details about a workflow. Please refer the [Tinkerbell CLI reference](cli/workflow.md) for the same. +The above command returns a UUID for the workflow thus created. +The workflow ID can be used for getting further details about a workflow. +Please refer the [Tinkerbell CLI reference](cli/workflow.md) for the same. -It's a good practice to verify that the worker have been well substituted in the template. In order to do so, use the following command: +It's a good practice to verify that the worker have been well substituted in the template. +In order to do so, use the following command: ```yaml $ tink workflow get From 977414678747eaa5f542fa746a1f5393e264aeea Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 4 Jun 2020 16:11:03 -0400 Subject: [PATCH 2/4] Add prettier to shell.nix --- shell.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/shell.nix b/shell.nix index 0556a5625..5b68f8290 100644 --- a/shell.nix +++ b/shell.nix @@ -13,6 +13,7 @@ mkShell { buildInputs = [ gnumake go + nodePackages.prettier pythonPackages.codespell shfmt shellcheck From e8f38486ca313c0edbbe9a3f5e6e865b43d8693d Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 4 Jun 2020 16:23:23 -0400 Subject: [PATCH 3/4] Cleanup up files with prettier --- README.md | 53 +++++++-------- deploy/docker-compose.yml | 12 ++-- deploy/tls/ca-config.json | 11 +--- docs/architecture.md | 3 +- docs/cli/README.md | 8 +-- docs/cli/hardware.md | 6 +- docs/cli/template.md | 56 ++++++++-------- docs/cli/workflow.md | 25 +++---- docs/concepts.md | 102 +++++++++++++++++------------ docs/hello-world.md | 87 +++++++++++++----------- docs/setup.md | 54 ++++++++------- docs/troubleshoot.md | 9 ++- docs/writing-workflow.md | 68 ++++++++++--------- test-docker-compose.yml | 17 +++-- test/data/hardware/hardware_2.json | 1 - 15 files changed, 273 insertions(+), 239 deletions(-) diff --git a/README.md b/README.md index ca235f393..5290a9753 100644 --- a/README.md +++ b/README.md @@ -9,35 +9,36 @@ The CLI is used to create a workflow along with its building blocks, templates a # Packet Workflow A Packet Workflow is an open-source microservice that’s responsible for handling flexible, bare metal provisioning workflows, that is... - - standalone and does not need the Packet API to function - - contains `Boots`, `Tinkerbell`, `Osie`, and workers - - can bootstrap any remote worker using `Boots + Osie` - - can run any set of actions as Docker container runtimes - - receive, manipulate, and save runtime data + +- standalone and does not need the Packet API to function +- contains `Boots`, `Tinkerbell`, `Osie`, and workers +- can bootstrap any remote worker using `Boots + Osie` +- can run any set of actions as Docker container runtimes +- receive, manipulate, and save runtime data ## Content - - [Setup](docs/setup.md) - - [Components](docs/components.md) - - [Boots](docs/components.md#boots) - - [Osie](docs/components.md#osie) - - [Tinkerbell](docs/components.md#tinkerbell) - - [Hegel](docs/components.md#hegel) - - [Database](docs/components.md#database) - - [Image Registry](docs/components.md#registry) - - [Elasticsearch](docs/components.md#elastic) - - [Fluent Bit](docs/components.md#fluent-bit) - - [Kibana](docs/components.md#kibana) - - [Architecture](docs/architecture.md) - - [Say "Hello-World!" with a Workflow](docs/hello-world.md) - - [Concepts](docs/concepts.md) - - [Template](docs/concepts.md#template) - - [Provisioner](docs/concepts.md#provisioner) - - [Worker](docs/concepts.md#worker) - - [Ephemeral Data](docs/concepts.md#ephemeral-data) - - [Writing a Workflow](docs/writing-workflow.md) - - [Tinkerbell CLI Reference](docs/cli/README.md) - - [Troubleshooting](docs/troubleshoot.md) +- [Setup](docs/setup.md) +- [Components](docs/components.md) + - [Boots](docs/components.md#boots) + - [Osie](docs/components.md#osie) + - [Tinkerbell](docs/components.md#tinkerbell) + - [Hegel](docs/components.md#hegel) + - [Database](docs/components.md#database) + - [Image Registry](docs/components.md#registry) + - [Elasticsearch](docs/components.md#elastic) + - [Fluent Bit](docs/components.md#fluent-bit) + - [Kibana](docs/components.md#kibana) +- [Architecture](docs/architecture.md) +- [Say "Hello-World!" with a Workflow](docs/hello-world.md) +- [Concepts](docs/concepts.md) + - [Template](docs/concepts.md#template) + - [Provisioner](docs/concepts.md#provisioner) + - [Worker](docs/concepts.md#worker) + - [Ephemeral Data](docs/concepts.md#ephemeral-data) +- [Writing a Workflow](docs/writing-workflow.md) +- [Tinkerbell CLI Reference](docs/cli/README.md) +- [Troubleshooting](docs/troubleshoot.md) ## Website diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 24cb88929..dd956b2f3 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2.1' +version: "2.1" services: certs: build: tls @@ -53,11 +53,11 @@ services: - 5432:5432 depends_on: fluentbit: - condition: service_started + condition: service_started healthcheck: test: ["CMD-SHELL", "pg_isready -U tinkerbell"] interval: 1s - timeout: 1s + timeout: 1s retries: 30 logging: driver: fluentd @@ -137,9 +137,9 @@ services: cacher: condition: service_started logging: - driver: fluentd - options: - tag: boots + driver: fluentd + options: + tag: boots ports: - $TINKERBELL_HOST_IP:80:80/tcp - 67:67/udp diff --git a/deploy/tls/ca-config.json b/deploy/tls/ca-config.json index 9c9e14d77..d6d666e7a 100644 --- a/deploy/tls/ca-config.json +++ b/deploy/tls/ca-config.json @@ -6,18 +6,11 @@ "profiles": { "server": { "expiry": "8760h", - "usages": [ - "signing", - "key encipherment", - "server auth" - ] + "usages": ["signing", "key encipherment", "server auth"] }, "signing": { "expiry": "8760h", - "usages": [ - "signing", - "key encipherment" - ] + "usages": ["signing", "key encipherment"] } } } diff --git a/docs/architecture.md b/docs/architecture.md index 3732da083..d63482807 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,4 +1,3 @@ # Architecture - -![](img/workflow-architecture.png) \ No newline at end of file +![](img/workflow-architecture.png) diff --git a/docs/cli/README.md b/docs/cli/README.md index 1cac870ad..bfe8ab634 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -27,8 +27,6 @@ It also enables you to create a template which is eventually used to create a wo ### See Also - - [tink hardware](hardware.md) - Hardware (worker) data operations - - [tink template](template.md) - Template operations - - [tink workflow](workflow.md) - Workflow operations - - +- [tink hardware](hardware.md) - Hardware (worker) data operations +- [tink template](template.md) - Template operations +- [tink workflow](workflow.md) - Workflow operations diff --git a/docs/cli/hardware.md b/docs/cli/hardware.md index 484544b59..e91511f8f 100644 --- a/docs/cli/hardware.md +++ b/docs/cli/hardware.md @@ -5,6 +5,7 @@ Hardware (worker) data operations. ### Synopsis Hardware operations: + ```shell all Get all known hardware for facility id Get hardware by id @@ -23,6 +24,5 @@ Hardware operations: ### See Also - - [tink template](template.md) - Template operations - - [tink workflow](workflow.md) - Workflow operations - +- [tink template](template.md) - Template operations +- [tink workflow](workflow.md) - Workflow operations diff --git a/docs/cli/template.md b/docs/cli/template.md index 044ce57e1..8eb907cac 100644 --- a/docs/cli/template.md +++ b/docs/cli/template.md @@ -5,6 +5,7 @@ Template operations. ### Synopsis Template operations: + ```shell create create a workflow template delete delete a template @@ -21,32 +22,35 @@ Template operations: ### Examples - - The following command creates a workflow template using the `sample.tmpl` file and save it as `sample`. - It returns a UUID for the newly created template. - ```shell - $ tink template create -n -p - $ tink template create -n sample -p /tmp/sample.tmpl - ``` - - - List all the templates - ```shell - $ tink template list - ``` - - - Update the name of an existing template - ```shell - $ tink template update -n - $ tink template update edb80a56-b1f2-4502-abf9-17326324192b -n new-sample-template - ``` - - - Update an existing template and keep the same name - ```shell - $ tink template update -p - $ tink template update edb80a56-b1f2-4502-abf9-17326324192b -p /tmp/new-sample-template.tmpl - ``` +- The following command creates a workflow template using the `sample.tmpl` file and save it as `sample`. + It returns a UUID for the newly created template. -### See Also +```shell + $ tink template create -n -p + $ tink template create -n sample -p /tmp/sample.tmpl +``` + +- List all the templates + +```shell + $ tink template list +``` + +- Update the name of an existing template - - [tink hardware](hardware.md) - Hardware (worker) data operations - - [tink workflow](workflow.md) - Workflow operations +```shell + $ tink template update -n + $ tink template update edb80a56-b1f2-4502-abf9-17326324192b -n new-sample-template +``` + +- Update an existing template and keep the same name + +```shell + $ tink template update -p + $ tink template update edb80a56-b1f2-4502-abf9-17326324192b -p /tmp/new-sample-template.tmpl +``` + +### See Also +- [tink hardware](hardware.md) - Hardware (worker) data operations +- [tink workflow](workflow.md) - Workflow operations diff --git a/docs/cli/workflow.md b/docs/cli/workflow.md index b6f955837..7d4af1589 100644 --- a/docs/cli/workflow.md +++ b/docs/cli/workflow.md @@ -5,6 +5,7 @@ Workflow operations. ### Synopsis Workflow operations: + ```shell create create a workflow data get workflow data @@ -23,18 +24,20 @@ Workflow operations: ### Examples - - Create a workflow using a template and hardware devices - ```shell - $ tink workflow create -t -r - $ tink workflow create -t edb80a56-b1f2-4502-abf9-17326324192b -r {"device_1": "mac/IP"} - ``` - #### Note: - 1. The key used in the above command which is "device_1" should be in sync with "worker" field in the template. +- Create a workflow using a template and hardware devices + +```shell + $ tink workflow create -t -r + $ tink workflow create -t edb80a56-b1f2-4502-abf9-17326324192b -r {"device_1": "mac/IP"} +``` + +#### Note: + +1. The key used in the above command which is "device_1" should be in sync with "worker" field in the template. Click [here](../concepts.md) to check the template structure. - 2. These keys can only contain letter, numbers and underscore. +2. These keys can only contain letter, numbers and underscore. ### See Also - - [tink hardware](hardware.md) - Hardware (worker) data operations - - [tink template](template.md) - Template operations - +- [tink hardware](hardware.md) - Hardware (worker) data operations +- [tink template](template.md) - Template operations diff --git a/docs/concepts.md b/docs/concepts.md index b219bba1d..539c3e497 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -1,7 +1,8 @@ # Concepts ### Hardware -A *hardware device* is defined separately and is substituted in a template at the time of creating a workflow. + +A _hardware device_ is defined separately and is substituted in a template at the time of creating a workflow. ### Template @@ -15,37 +16,37 @@ A user can CRUD a template using the CLI (`tink template`). Here is a sample workflow template: ```yaml -version: '0.1' +version: "0.1" name: ubuntu_provisioning global_timeout: 6000 tasks: -- name: "os-installation" - worker: "{{.device_1}}" - volumes: - - /dev:/dev - - /dev/console:/dev/console - - /lib/firmware:/lib/firmware:ro - environment: - MIRROR_HOST: 192.168.1.2 - actions: - - name: "disk-wipe" - image: disk-wipe - timeout: 90 - - name: "disk-partition" - image: disk-partition - timeout: 600 - environment: - MIRROR_HOST: 192.168.1.3 - volumes: - - /statedir:/statedir - - name: "install-root-fs" - image: install-root-fs - timeout: 600 - - name: "install-grub" - image: install-grub - timeout: 600 - volumes: - - /statedir:/statedir + - name: "os-installation" + worker: "{{.device_1}}" + volumes: + - /dev:/dev + - /dev/console:/dev/console + - /lib/firmware:/lib/firmware:ro + environment: + MIRROR_HOST: 192.168.1.2 + actions: + - name: "disk-wipe" + image: disk-wipe + timeout: 90 + - name: "disk-partition" + image: disk-partition + timeout: 600 + environment: + MIRROR_HOST: 192.168.1.3 + volumes: + - /statedir:/statedir + - name: "install-root-fs" + image: install-root-fs + timeout: 600 + - name: "install-grub" + image: install-grub + timeout: 600 + volumes: + - /statedir:/statedir ``` A template comprises Tasks, which are executed in a sequential manner. @@ -58,20 +59,20 @@ Therefore, any entry at an action will overwrite the value defined at the task l For example, in the above template the `MIRROR_HOST` environment variable defined at action `disk-partition` will overwrite the value defined at task level. However, the other actions will receive the original value defined at the task level. - ### Provisioner The provisioner machine is the main driver for executing a workflow. A provisioner houses the following components: - - Database (Postgres) - - Tinkerbell (CLI and server) - - Boots - - Hegel - - Image Registry (optional) - - Elasticsearch - - Fluent Bit - - Kibana - - NGINX + +- Database (Postgres) +- Tinkerbell (CLI and server) +- Boots +- Hegel +- Image Registry (optional) +- Elasticsearch +- Fluent Bit +- Kibana +- NGINX It is up to you if you would like to divide these components into multiple servers. @@ -84,7 +85,6 @@ When the node boots, a worker container starts and connects with provisioner to After the completion of an action, the worker sends action status to provisioner. When all workflows which are related to a worker are complete, a worker can terminate. - ### Ephemeral Data The workers that are part of a workflow might require to share some data. @@ -92,13 +92,29 @@ This can take the form of a light JSON like below, or some binary files that oth For instance, a worker may add the following data: ```json - {"operating_system": "ubuntu_18_04", "mac_addr": "F5:C9:E2:99:BD:9B", "instance_id": "123e4567-e89b-12d3-a456-426655440000"} +{ + "instance_id": "123e4567-e89b-12d3-a456-426655440000", + "mac_addr": "F5:C9:E2:99:BD:9B", + "operating_system": "ubuntu_18_04" +} ``` The other worker may retrieve and use this data and eventually add some more: ```json -{"operating_system": "ubuntu_18_04", "mac_addr": "F5:C9:E2:99:BD:9B", "instance_id": "123e4567-e89b-12d3-a456-426655440000", "ip_addresses": [{"address_family": 4, "address": "172.27.0.23", "cidr": 31, "private": true}]} +{ + "instance_id": "123e4567-e89b-12d3-a456-426655440000", + "ip_addresses": [ + { + "address": "172.27.0.23", + "address_family": 4, + "cidr": 31, + "private": true + } + ], + "mac_addr": "F5:C9:E2:99:BD:9B", + "operating_system": "ubuntu_18_04" +} ``` -![](img/ephemeral-data.png) +![](img/ephemeral-data.png) diff --git a/docs/hello-world.md b/docs/hello-world.md index a5b0c5970..ce6c48614 100644 --- a/docs/hello-world.md +++ b/docs/hello-world.md @@ -10,49 +10,53 @@ If not, please follow the steps [here](setup.md) to complete the setup. ### Hardware Data While the data model changes are in progress, the following data should be enough to get your workflow rolling at the moment: + ```json { - "id": "ce2e62ed-826f-4485-a39f-a82bb74338e2", - "arch": "x86_64", - "allow_pxe": true, - "allow_workflow": true, - "facility_code": "onprem", - "ip_addresses": [ - { - "address": "192.168.1.5", - "address_family": 4, - "enabled": true, - "gateway": "192.168.1.1", - "management": true, - "netmask": "255.255.255.248", - "public": false - } - ], - "network_ports": [ - { - "data": { - "mac": "ec:0d:9a:bf:ff:dc" - }, - "name": "eth0", - "type": "data" - } - ] + "allow_pxe": true, + "allow_workflow": true, + "arch": "x86_64", + "facility_code": "onprem", + "id": "ce2e62ed-826f-4485-a39f-a82bb74338e2", + "ip_addresses": [ + { + "address": "192.168.1.5", + "address_family": 4, + "enabled": true, + "gateway": "192.168.1.1", + "management": true, + "netmask": "255.255.255.248", + "public": false + } + ], + "network_ports": [ + { + "data": { + "mac": "ec:0d:9a:bf:ff:dc" + }, + "name": "eth0", + "type": "data" + } + ] } ``` A few details to note: - - `id` is the hardware UUID - - `allow_workflow` must be `true` to boot into workflow mode - - `ip_addresses.address` is the IP to hand out - - `network_ports` is the worker MAC address + +- `id` is the hardware UUID +- `allow_workflow` must be `true` to boot into workflow mode +- `ip_addresses.address` is the IP to hand out +- `network_ports` is the worker MAC address Now that we have the hardware data, we need to push it into the database. In order to do so, remove the extra spaces in the above JSON and use the following command to push the data: + ``` $ tink hardware push '' ``` Verify that the data was actually pushed using the command: + ```shell $ tink hardware mac ``` @@ -61,6 +65,7 @@ $ tink hardware mac The workflow will have a single task that will have a single action. The action here is to say `Hello-world!`, so we will push the action image to the registry running on the provisioner: + ```shell $ docker pull hello-world $ docker tag hello-world /hello-world @@ -70,14 +75,18 @@ $ docker push /hello-world ### Workflow We can now define a workflow with the following steps: - - Create a template: - ```shell - # get the template from examples/hello-world.tmpl and save it - $ tink template create -n hello-world -p hello-world.tmpl - ``` - - Create a workflow: - ```shell - $ tink workflow create -t -r '{"device_1":"mac/IP"}' - ``` - - Reboot the worker machine +- Create a template: + +```shell + # get the template from examples/hello-world.tmpl and save it + $ tink template create -n hello-world -p hello-world.tmpl +``` + +- Create a workflow: + +```shell + $ tink workflow create -t -r '{"device_1":"mac/IP"}' +``` + +- Reboot the worker machine diff --git a/docs/setup.md b/docs/setup.md index d7a07249a..172b4c9c3 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -1,19 +1,24 @@ # Setup the Provisioner ### Prerequisites - - The setup must be executed as a privileged or root user. - - The setup downloads about 1.5GB of assets, therefore, it is advised to have a minimum of 10GB disk space available before you begin. + +- The setup must be executed as a privileged or root user. +- The setup downloads about 1.5GB of assets, therefore, it is advised to have a minimum of 10GB disk space available before you begin. ### Interactive Mode + Execute the following commands to bring up the Tinkerbell stack with an interactive setup: + ```shell $ wget https://raw.githubusercontent.com/tinkerbell/tink/master/setup.sh && chmod +x setup.sh $ ./setup.sh ``` ### Declarative Mode + You can also execute the setup in declarative mode. In order to do so, define the following environment variables (examples here): + ```shell export TB_INTERFACE=network-interface # enp1s0f1 export TB_NETWORK=network-with-cidr # 192.168.1.0/29 @@ -22,37 +27,42 @@ export TB_REGUSER=registry-username # admin ``` Now, you can execute the setup with the following command: + ```shell $ curl https://raw.githubusercontent.com/tinkerbell/tink/master/setup.sh | bash ``` ### Good to know - - All the environment variables are kept in the `envrc` file, which is generated from the setup itself. - - It is advised that you keep all the environment variables in the same file. - - It is important to note that if you execute the setup the again, a new `envrc` will be generated. - However, the existing environment configuration be saved as `envrc.bak`. - - The setup removes all the `.tar.gz` files downloaded in process. + +- All the environment variables are kept in the `envrc` file, which is generated from the setup itself. +- It is advised that you keep all the environment variables in the same file. +- It is important to note that if you execute the setup the again, a new `envrc` will be generated. + However, the existing environment configuration be saved as `envrc.bak`. +- The setup removes all the `.tar.gz` files downloaded in process. ### For Packet Environment The script was tested with: - - Server class: `c3.small.x86` - - Operating System: Ubuntu 18.04 and CentOS 7 - - Region: Amsterdam, NL (AMS1) - - ENV variables: - ```shell - export TB_INTERFACE=enp1s0f1 - export TB_NETWORK=192.168.1.0/29 - export TB_IPADDR=192.168.1.1 - export TB_REGUSER=admin - ``` - - Command: - ```shell - $ curl https://raw.githubusercontent.com/tinkerbell/tink/master/setup.sh | bash - ``` + +- Server class: `c3.small.x86` +- Operating System: Ubuntu 18.04 and CentOS 7 +- Region: Amsterdam, NL (AMS1) +- ENV variables: + +```shell + export TB_INTERFACE=enp1s0f1 + export TB_NETWORK=192.168.1.0/29 + export TB_IPADDR=192.168.1.1 + export TB_REGUSER=admin +``` + +- Command: + +```shell +$ curl https://raw.githubusercontent.com/tinkerbell/tink/master/setup.sh | bash +``` ### What's Next Once you have the provisioner setup successfully, you can try executing your first workflow. Follow the steps described in [here](hello-world.md) to say "Hello World!" with a workflow. - diff --git a/docs/troubleshoot.md b/docs/troubleshoot.md index b6e346dc6..7e8f937f5 100644 --- a/docs/troubleshoot.md +++ b/docs/troubleshoot.md @@ -1,6 +1,5 @@ # Troubleshooting - ### Not setting the worker to always PXE ```shell @@ -11,6 +10,7 @@ pre-boot environment. Enable UEFI boot from the BIOS settings. To enable PXE boot, execute the following commands: + ```shell $ apk add ipmitool $ ipmitool chassis bootdev pxe @@ -36,8 +36,8 @@ You can push the data with `tink push `. Login did not succeed, error: Error response from daemon: Get https://192.168.1.1/v2/: x509: certificate signed by unknown authority (possibly because of "crypto/rsa: verification error" while trying to verify candidate authority certificate "Autogenerated CA") ``` - - verify that `hosts` list in `tinkerbell/tls/server-csr.in.json` has the registry IP or domain - - rebuild the `certs` container and copy the `ca.pem` to `/etc/docker/certs.d//ca.crt` +- verify that `hosts` list in `tinkerbell/tls/server-csr.in.json` has the registry IP or domain +- rebuild the `certs` container and copy the `ca.pem` to `/etc/docker/certs.d//ca.crt` ### Worker fails to pull images from private registry @@ -46,5 +46,4 @@ Waiting for docker to respond... Error response from daemon: Get https://192.168.1.1/v2/: x509: certificate signed by unknown authority (possibly because of "crypto/rsa: verification error" while trying to verify candidate authority certificate "Autogenerated CA") ``` -Make sure that the `ca.pem` file, used for hosting the private registry, is present at `/misc/tinkerbell/workflow` path. - +Make sure that the `ca.pem` file, used for hosting the private registry, is present at `/misc/tinkerbell/workflow` path. diff --git a/docs/writing-workflow.md b/docs/writing-workflow.md index 38bee8ced..6893562a6 100644 --- a/docs/writing-workflow.md +++ b/docs/writing-workflow.md @@ -6,50 +6,53 @@ Any workflow comprises two building blocks: worker (targeted hardware) and a tem A worker is targeted hardware on which workflow needs to run. User need to push the hardware details as per the below command: + ```shell $ tink hardware push "" ``` + ### Creating a [template](concepts.md#template) Consider a sample template like the following saved as `/tmp/sample.tmpl`. ```yaml -version: '0.1' +version: "0.1" name: ubuntu_provisioning global_timeout: 2500 tasks: -- name: "os-installation" - worker: "{{.device_1}}" - volumes: - - /dev:/dev - - /lib/firmware:/lib/firmware:ro - environment: - MIRROR_HOST: - actions: - - name: "disk-partition" - image: disk-partition - timeout: 600 - environment: - MIRROR_HOST: - volumes: - - /statedir:/statedir - - name: "install-root-fs" - image: install-root-fs - timeout: 600 - - name: "install-grub" - image: install-grub - timeout: 600 - volumes: - - /statedir:/statedir + - name: "os-installation" + worker: "{{.device_1}}" + volumes: + - /dev:/dev + - /lib/firmware:/lib/firmware:ro + environment: + MIRROR_HOST: + actions: + - name: "disk-partition" + image: disk-partition + timeout: 600 + environment: + MIRROR_HOST: + volumes: + - /statedir:/statedir + - name: "install-root-fs" + image: install-root-fs + timeout: 600 + - name: "install-grub" + image: install-grub + timeout: 600 + volumes: + - /statedir:/statedir ``` Key points: - - `global_timeout` is in seconds. - - Any worflow that exceeds the global timeout will be terminated and marked as failed. - - Each action has its own timeout. - If an action reaches its timeout, it is marked as failed and so is the workflow. - - An action cannot have space (` `) in its name. - - Environment variables and volumes at action level overwrites the values for duplicate keys defined at task level. + +- `global_timeout` is in seconds. +- Any worflow that exceeds the global timeout will be terminated and marked as failed. +- Each action has its own timeout. + If an action reaches its timeout, it is marked as failed and so is the workflow. +- An action cannot have space (``) in its name. +- Environment variables and volumes at action level overwrites the values for duplicate keys defined at task level. A worker can be accessed in a template like: @@ -59,14 +62,15 @@ A worker can be accessed in a template like: ``` The following command creates a workflow template and returns a UUID: + ```shell $ tink template create -n sample -p /tmp/sample.tmpl ``` - ### Creating a [workflow](concepts.md#workflow) We can create a workflow using the above created (or existing) template and worker. + ```shell $ tink workflow create -t -r '{worker machines in json format}' $ tink workflow create -t edb80a56-b1f2-4502-abf9-17326324192b -r '{"device_1":"mac/IP"}' @@ -78,6 +82,7 @@ Please refer the [Tinkerbell CLI reference](cli/workflow.md) for the same. It's a good practice to verify that the worker have been well substituted in the template. In order to do so, use the following command: + ```yaml $ tink workflow get @@ -111,4 +116,3 @@ tasks: ``` Notice how `worker` is set to the MAC address we had defined in the input while creating a workflow. - diff --git a/test-docker-compose.yml b/test-docker-compose.yml index 4aa29563c..e31c410d0 100644 --- a/test-docker-compose.yml +++ b/test-docker-compose.yml @@ -1,4 +1,4 @@ -version: '2.1' +version: "2.1" services: certs: build: tls @@ -7,7 +7,7 @@ services: tinkerbell: build: . - environment: + environment: FACILITY: ${FACILITY:-lab1} PACKET_ENV: ${PACKET_ENV:-testing} PACKET_VERSION: ${PACKET_VERSION:-5efab5ef3a42cb88f2d54f4ed3201c2dd6797b7d} @@ -31,7 +31,7 @@ services: interval: 5s timeout: 2s retries: 30 - volumes: + volumes: - ./certs:/certs/${FACILITY} #logging: #driver: fluentd @@ -62,21 +62,21 @@ services: driver: fluentd options: tag: db - + registry: build: context: registry - args: + args: REGISTRY_USERNAME: username REGISTRY_PASSWORD: password - environment: + environment: REGISTRY_HTTP_ADDR: localhost:443 REGISTRY_HTTP_TLS_CERTIFICATE: /certs/server.pem REGISTRY_HTTP_TLS_KEY: /certs/server-key.pem REGISTRY_AUTH: htpasswd REGISTRY_AUTH_HTPASSWD_REALM: "Registry Realm" REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd - volumes: + volumes: - ./certs:/certs depends_on: - fluentbit @@ -127,7 +127,7 @@ services: - 127.0.0.1:80:80/tcp - 67:67/udp - 69:69/udp - + elasticsearch: image: elasticsearch:7.3.0 ports: @@ -156,4 +156,3 @@ services: - kibana volumes: - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf - diff --git a/test/data/hardware/hardware_2.json b/test/data/hardware/hardware_2.json index 0bbb71da2..fb85ab269 100644 --- a/test/data/hardware/hardware_2.json +++ b/test/data/hardware/hardware_2.json @@ -110,4 +110,3 @@ "type": "server", "vlan_id": null } - From 91a82fcf2eea516082cd2ca972cbb6703fc50d7e Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 4 Jun 2020 16:28:23 -0400 Subject: [PATCH 4/4] Add prettier check to ci-check.sh and thus ci --- .drone.yml | 2 +- ci-checks.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 2949f8a02..773143dd1 100644 --- a/.drone.yml +++ b/.drone.yml @@ -56,7 +56,7 @@ steps: commands: - CGO_ENABLED=0 go test -v ./... - - name: check spelling + - name: ci-checks image: nixos/nix commands: - ./ci-checks.sh diff --git a/ci-checks.sh b/ci-checks.sh index d84b9a504..eccfcdf86 100755 --- a/ci-checks.sh +++ b/ci-checks.sh @@ -2,3 +2,4 @@ #!nix-shell -i bash codespell -q 3 -I .codespell-whitelist * +prettier --check '**/*.json' '**/*.md' '**/*.yml'