Add configurable working directory (#1231)

awslabs · Jun 16, 2023 · 6412bee · 6412bee
1 parent 05f1146
commit 6412bee
Show file tree

Hide file tree

Showing 6 changed files with 165 additions and 48 deletions.
diff --git a/doc/USER_GUIDE.md b/doc/USER_GUIDE.md
@@ -23,9 +23,58 @@ Default values for most variables are defined in [a default variable file](eks-w
 Users have the following options for specifying their own values:
 
 1. Provide a variable file with the `PACKER_VARIABLE_FILE` argument to `make`. Values in this file will override values in the default variable file. Your variable file does not need to include all possible variables, as it will be merged with the default variable file.
-2. Pass a key-value pair for any template variable to `make`. These values will override any values that were specified with the first method.
+2. Pass a key-value pair for any template variable to `make`. These values will override any values that were specified with the first method. In the table below, these variables have a default value of "None".
 
-**Note** that some variables (such as `arch` and `kubernetes_version`) do not have a sensible, static default, and are satisfied by the Makefile. Such variables do not appear in the default variable file, and must be overridden (if necessary) by the second method described above.
+> **Note**
+> Some variables (such as `arch` and `kubernetes_version`) do not have a sensible, static default, and are satisfied by the Makefile.
+> Such variables do not appear in the default variable file, and must be overridden (if necessary) by the second method described above.
+
+<!-- this table is generated by hack/generate-template-variable-doc.sh -->
+<!-- template-variable-table-boundary -->
+| Variable | Default value | Description |
+| - | - | - |
+| `additional_yum_repos` | `""` |  |
+| `ami_component_description` | ```{{user `remote_folder`}}/worker``` |  |
+| `ami_description` | ```{{user `remote_folder`}}/worker``` |  |
+| `ami_name` | None |  |
+| `ami_regions` | `""` |  |
+| `ami_users` | `""` |  |
+| `arch` | None |  |
+| `associate_public_ip_address` | `""` |  |
+| `aws_access_key_id` | ```{{user `remote_folder`}}/worker``` |  |
+| `aws_region` | ```{{user `remote_folder`}}/worker``` |  |
+| `aws_secret_access_key` | ```{{user `remote_folder`}}/worker``` |  |
+| `aws_session_token` | ```{{user `remote_folder`}}/worker``` |  |
+| `binary_bucket_name` | ```{{user `remote_folder`}}/worker``` |  |
+| `binary_bucket_region` | ```{{user `remote_folder`}}/worker``` |  |
+| `cache_container_images` | ```{{user `remote_folder`}}/worker``` |  |
+| `cni_plugin_version` | ```{{user `remote_folder`}}/worker``` |  |
+| `containerd_version` | ```{{user `remote_folder`}}/worker``` |  |
+| `creator` | ```{{user `remote_folder`}}/worker``` |  |
+| `docker_version` | ```{{user `remote_folder`}}/worker``` |  |
+| `encrypted` | ```{{user `remote_folder`}}/worker``` |  |
+| `instance_type` | None |  |
+| `kernel_version` | `""` |  |
+| `kms_key_id` | `""` |  |
+| `kubernetes_build_date` | None |  |
+| `kubernetes_version` | None |  |
+| `launch_block_device_mappings_volume_size` | ```{{user `remote_folder`}}/worker``` |  |
+| `pause_container_version` | ```{{user `remote_folder`}}/worker``` |  |
+| `pull_cni_from_github` | ```{{user `remote_folder`}}/worker``` |  |
+| `remote_folder` | ```{{user `remote_folder`}}/worker``` | Directory path for shell provisioner scripts on the builder instance |
+| `runc_version` | ```{{user `remote_folder`}}/worker``` |  |
+| `security_group_id` | `""` |  |
+| `sonobuoy_e2e_registry` | `""` |  |
+| `source_ami_filter_name` | ```{{user `remote_folder`}}/worker``` |  |
+| `source_ami_id` | `""` |  |
+| `source_ami_owners` | ```{{user `remote_folder`}}/worker``` |  |
+| `ssh_interface` | `""` |  |
+| `ssh_username` | ```{{user `remote_folder`}}/worker``` |  |
+| `subnet_id` | `""` |  |
+| `temporary_security_group_source_cidrs` | `""` |  |
+| `volume_type` | ```{{user `remote_folder`}}/worker``` |  |
+| `working_dir` | ```{{user `remote_folder`}}/worker``` | Directory path for ephemeral resources on the builder instance |
+<!-- template-variable-table-boundary -->
 
 ---
 

diff --git a/eks-worker-al2-variables.json b/eks-worker-al2-variables.json
@@ -22,7 +22,7 @@
     "launch_block_device_mappings_volume_size": "4",
     "pause_container_version": "3.5",
     "pull_cni_from_github": "true",
-    "remote_folder": "",
+    "remote_folder": "/tmp",
     "runc_version": "1.1.5-1.amzn2",
     "security_group_id": "",
     "sonobuoy_e2e_registry": "",
@@ -33,5 +33,6 @@
     "ssh_username": "ec2-user",
     "subnet_id": "",
     "temporary_security_group_source_cidrs": "",
-    "volume_type": "gp2"
+    "volume_type": "gp2",
+    "working_dir": "{{user `remote_folder`}}/worker"
 }
diff --git a/eks-worker-al2.json b/eks-worker-al2.json
@@ -40,7 +40,8 @@
     "ssh_username": null,
     "subnet_id": null,
     "temporary_security_group_source_cidrs": null,
-    "volume_type": null
+    "volume_type": null,
+    "working_dir": null
   },
   "builders": [
     {
@@ -114,34 +115,34 @@
   "provisioners": [
     {
       "type": "shell",
-      "remote_folder": "{{ user `remote_folder`}}",
-      "script": "{{template_dir}}/scripts/install_additional_repos.sh",
-      "environment_vars": [
-        "ADDITIONAL_YUM_REPOS={{user `additional_yum_repos`}}"
+      "inline": [
+        "mkdir -p {{user `working_dir`}}",
+        "mkdir -p {{user `working_dir`}}/log-collector-script"
       ]
     },
     {
       "type": "shell",
       "remote_folder": "{{ user `remote_folder`}}",
-      "inline": [
-        "mkdir -p /tmp/worker/log-collector-script/"
+      "script": "{{template_dir}}/scripts/install_additional_repos.sh",
+      "environment_vars": [
+        "ADDITIONAL_YUM_REPOS={{user `additional_yum_repos`}}"
       ]
     },
     {
       "type": "file",
       "source": "{{template_dir}}/files/",
-      "destination": "/tmp/worker/"
+      "destination": "{{user `working_dir`}}"
     },
     {
       "type": "file",
       "source": "{{template_dir}}/log-collector-script/linux/",
-      "destination": "/tmp/worker/log-collector-script/"
+      "destination": "{{user `working_dir`}}/log-collector-script/"
     },
     {
       "type": "shell",
       "inline": [
-        "sudo chmod -R a+x /tmp/worker/bin/",
-        "sudo mv /tmp/worker/bin/* /usr/bin/"
+        "sudo chmod -R a+x {{user `working_dir`}}/bin/",
+        "sudo mv {{user `working_dir`}}/bin/* /usr/bin/"
       ]
     },
     {
@@ -174,7 +175,8 @@
         "AWS_SESSION_TOKEN={{user `aws_session_token`}}",
         "SONOBUOY_E2E_REGISTRY={{user `sonobuoy_e2e_registry`}}",
         "PAUSE_CONTAINER_VERSION={{user `pause_container_version`}}",
-        "CACHE_CONTAINER_IMAGES={{user `cache_container_images`}}"
+        "CACHE_CONTAINER_IMAGES={{user `cache_container_images`}}",
+        "WORKING_DIR={{user `working_dir`}}"
       ]
     },
     {
@@ -202,13 +204,19 @@
       "type": "shell",
       "remote_folder": "{{ user `remote_folder`}}",
       "script": "{{template_dir}}/scripts/generate-version-info.sh",
-      "execute_command": "chmod +x {{ .Path }}; {{ .Path }} /tmp/version-info.json"
+      "execute_command": "chmod +x {{ .Path }}; {{ .Path }} {{user `working_dir`}}/version-info.json"
     },
     {
       "type": "file",
       "direction": "download",
-      "source": "/tmp/version-info.json",
+      "source": "{{user `working_dir`}}/version-info.json",
       "destination": "{{ user `ami_name` }}-version-info.json"
+    },
+    {
+      "type": "shell",
+      "inline": [
+        "rm -rf {{user `working_dir`}}"
+      ]
     }
   ],
   "post-processors": [

diff --git a/hack/generate-template-variable-doc.py b/hack/generate-template-variable-doc.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import re
+
+whereami = os.path.abspath(__file__)
+os.chdir(os.path.dirname(whereami))
+
+template = {}
+with open('../eks-worker-al2.json') as template_file:
+    template = json.load(template_file)
+
+default_vars = {}
+with open('../eks-worker-al2-variables.json') as default_var_file:
+    default_vars = json.load(default_var_file)
+
+all_vars = {}
+
+for var in template['variables']:
+    all_vars[var] = None
+for var, default_val in default_vars.items():
+    all_vars[var] = default_val
+
+doc_file_name = '../doc/USER_GUIDE.md'
+doc = None
+with open(doc_file_name) as doc_file:
+    doc = doc_file.read()
+
+table_boundary = '<!-- template-variable-table-boundary -->'
+existing_table_pattern = f"{table_boundary}([\S\s]*){table_boundary}"
+existing_table_matches = re.search(existing_table_pattern, doc)
+existing_table_lines = existing_table_matches.group(1).splitlines()
+
+new_table = f"{table_boundary}\n"
+new_table += f"{existing_table_lines[1]}\n"
+new_table += f"{existing_table_lines[2]}\n"
+
+existing_descriptions = {}
+for line in existing_table_lines[3:]:
+    columns = line.split('|')
+    var = columns[1].strip(" `")
+    existing_descriptions[var] = columns[3].strip(" `")
+
+for var, val in all_vars.items():
+    if val is not None:
+        if val == "":
+            val = f"`\"\"`"
+        else:
+            val = f"```{default_val}```"
+    description = ""
+    if var in existing_descriptions:
+        description = existing_descriptions[var]
+    new_table += f"| `{var}` | {val} | {description} |\n"
+
+new_table += table_boundary
+
+replace_doc_pattern = f"{table_boundary}[\S\s]*{table_boundary}"
+new_doc = re.sub(replace_doc_pattern, new_table, doc)
+
+with open(doc_file_name, 'w') as doc_file:
+    doc_file.write(new_doc)
diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh
@@ -4,9 +4,6 @@
 sudo yum clean all
 sudo rm -rf /var/cache/yum
 
-# Clean up build artifacts
-sudo rm -rf /tmp/worker
-
 # Clean up files to reduce confusion during debug
 sudo rm -rf \
   /etc/hostname \