From 0930020c14a3de1488700b8d6ff0fde196a437eb Mon Sep 17 00:00:00 2001 From: Christopher Tubbs Date: Mon, 26 Aug 2019 23:38:12 -0400 Subject: [PATCH] Miscellaneous trivial fixes * Add missing newlines at end of files * Remove trailing whitespace * Remove redundant Apache license headers * Make .gitignore files more specific * Add example pom.xml for contrib/add-license-headers.md * Format markdown files with syntax highlighting for code blocks * Format json files with jq * Add missing quotes for json templates * Add missing punctuation in READMEs (missing periods on sentences) * Add missing quotes in JSON in README --- .gitignore | 4 +- README.md | 14 +- ansible/.gitignore | 2 +- ansible/conf/.gitignore | 4 +- ansible/group_vars/.gitignore | 2 +- ansible/roles/accumulo/tasks/main.yml | 4 +- .../templates/log4j-service.properties | 17 ++ ansible/roles/common/tasks/main.yml | 2 +- ansible/roles/common/tasks/ssh.yml | 2 +- .../roles/fluo/templates/fluo-app.properties | 4 +- .../roles/fluo/templates/fluo-conn.properties | 2 +- ansible/roles/fluo/templates/fluo.properties | 2 +- .../fluo_yarn/templates/fluo-yarn.properties | 2 +- ansible/roles/grafana/tasks/main.yml | 4 +- ansible/roles/hadoop/templates/core-site.xml | 2 +- ansible/roles/hadoop/templates/hdfs-site.xml | 17 +- .../roles/hadoop/templates/mapred-site.xml | 4 +- ansible/roles/hadoop/templates/yarn-site.xml | 15 +- ansible/roles/influxdb/tasks/main.yml | 2 +- ansible/roles/proxy/tasks/download.yml | 2 +- ansible/roles/proxy/tasks/main.yml | 2 +- .../roles/spark/tasks/start-spark-history.yml | 2 +- ansible/roles/zookeeper/templates/zoo.cfg | 6 +- conf/.gitignore | 8 +- conf/templates/README.md | 180 +++++++++--------- conf/templates/example/accumulomaster.json | 64 +++---- conf/templates/example/client.json | 64 +++---- conf/templates/example/devices | 2 +- conf/templates/example/metrics.json | 64 +++---- conf/templates/example/namenode.json | 64 +++---- conf/templates/example/resourcemanager.json | 64 +++---- conf/templates/example/worker.json | 112 +++++------ conf/templates/example/zookeeper.json | 64 +++---- conf/templates/user/.gitignore | 2 +- conf/upload/.gitignore | 4 +- contrib/add-license-headers.md | 9 +- contrib/license-header.txt | 2 +- contrib/muchos-logo-dark.svg | 5 +- contrib/muchos-logo.svg | 4 +- contrib/pom.xml | 28 +++ lib/.gitignore | 2 +- 41 files changed, 440 insertions(+), 419 deletions(-) create mode 100644 contrib/pom.xml diff --git a/.gitignore b/.gitignore index a2f1c6db..3c468ed4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ *.pyc -.idea/ -pom.xml +/.idea/ +/pom.xml diff --git a/README.md b/README.md index 8a9f9f33..cfc8b0d0 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Muchos requires the following: The following commands will install Muchos, launch an EC2 cluster, and setup/run Accumulo: -```bash +```sh git clone https://github.com/apache/fluo-muchos.git cd fluo-muchos/ cp conf/muchos.props.example conf/muchos.props @@ -49,13 +49,13 @@ The `setup` command can be run repeatedly to fix any failures and will not repea After your cluster is launched, SSH to it using the following command: -```bash +```sh ./bin/muchos ssh ``` Run the following command to terminate your cluster. WARNING: All cluster data will be lost. -```bash +```sh ./bin/muchos terminate ``` @@ -167,7 +167,7 @@ Running an example Fluo application like [WebIndex], [Phrasecount], or [Stresso] with Muchos as it configures your shell with common environment variables. To run an example application, SSH to a node on cluster where Fluo is installed and clone the example repo: -```bash +```sh ./bin/muchos ssh # SSH to cluster proxy node ssh # Nodes with Fluo installed is determined by Muchos config hub clone apache/fluo-examples # Clone repo of Fluo example applications. Press enter for user/password. @@ -177,7 +177,7 @@ Start the example application using its provided scripts. To show how simple th to run the [WebIndex] application are shown below. Read the [WebIndex] README to learn more before running these commands. -```bash +```sh cd fluo-examples/webindex ./bin/webindex init # Initialize and start webindex Fluo application ./bin/webindex getpaths 2015-18 # Retrieves CommonCrawl paths file for 2015-18 crawl @@ -212,7 +212,7 @@ With the default configuration, EC2 clusters will not shutdown automatically aft shutdown behavior will be stopping the node. If you would like your cluster to terminate after 8 hours, set the following configuration in [muchos.props]: -``` +```ini shutdown_delay_minutes = 480 shutdown_behavior = terminate ``` @@ -223,7 +223,7 @@ If you decide later to cancel the shutdown, run `muchos cancel_shutdown`. The `config` command allows you to retrieve cluster configuration for your own scripts: -```bash +```sh $ ./bin/muchos config -p leader.public.ip 10.10.10.10 ``` diff --git a/ansible/.gitignore b/ansible/.gitignore index b39a1432..e23687b1 100644 --- a/ansible/.gitignore +++ b/ansible/.gitignore @@ -1 +1 @@ -site.yml +/site.yml diff --git a/ansible/conf/.gitignore b/ansible/conf/.gitignore index 55d232da..cc136676 100644 --- a/ansible/conf/.gitignore +++ b/ansible/conf/.gitignore @@ -1,2 +1,2 @@ -hosts -keys +/hosts +/keys diff --git a/ansible/group_vars/.gitignore b/ansible/group_vars/.gitignore index 0702cb5b..244b0c6e 100644 --- a/ansible/group_vars/.gitignore +++ b/ansible/group_vars/.gitignore @@ -1 +1 @@ -all +/all diff --git a/ansible/roles/accumulo/tasks/main.yml b/ansible/roles/accumulo/tasks/main.yml index 241ab7bc..b52851c9 100644 --- a/ansible/roles/accumulo/tasks/main.yml +++ b/ansible/roles/accumulo/tasks/main.yml @@ -56,8 +56,8 @@ - name: "configure tservers using managed templates" template: src=tservers dest={{ accumulo_home }}/conf/{{ accumulo_tservers_fn[accumulo_major_version] }} - name: "build accumulo native libraries" - shell: "{{ accumulo_build_native_cmd[accumulo_major_version] }}" - args: + shell: "{{ accumulo_build_native_cmd[accumulo_major_version] }}" + args: creates: "{{ accumulo_home }}/lib/native/libaccumulo.so" - name: "Create accumulo log dir" file: path={{ worker_data_dirs[0] }}/logs/accumulo state=directory diff --git a/ansible/roles/accumulo/templates/log4j-service.properties b/ansible/roles/accumulo/templates/log4j-service.properties index c09d46b5..5f255eab 100644 --- a/ansible/roles/accumulo/templates/log4j-service.properties +++ b/ansible/roles/accumulo/templates/log4j-service.properties @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + log4j.appender.file=org.apache.log4j.RollingFileAppender log4j.appender.file.File=${accumulo.log.dir}/${accumulo.application}.log log4j.appender.file.MaxFileSize=1000MB diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml index 93a4be43..185aae53 100644 --- a/ansible/roles/common/tasks/main.yml +++ b/ansible/roles/common/tasks/main.yml @@ -15,7 +15,7 @@ # limitations under the License. # -- name: "enable epel yum repo" +- name: "enable epel yum repo" yum: name=epel-release state=present - name: "install packages" yum: diff --git a/ansible/roles/common/tasks/ssh.yml b/ansible/roles/common/tasks/ssh.yml index 0e2e2afe..77e8af80 100644 --- a/ansible/roles/common/tasks/ssh.yml +++ b/ansible/roles/common/tasks/ssh.yml @@ -15,7 +15,7 @@ # limitations under the License. # -- name: "ensure cluster user exists but don't generate ssh key" +- name: "ensure cluster user exists but don't generate ssh key" user: name={{ cluster_user }} generate_ssh_key=no state=present - name: "copy cluster private key to all nodes" copy: src=/home/{{ cluster_user }}/.ssh/id_rsa dest=/home/{{ cluster_user }}/.ssh/id_rsa owner={{ cluster_user }} group={{ cluster_group }} mode=0600 diff --git a/ansible/roles/fluo/templates/fluo-app.properties b/ansible/roles/fluo/templates/fluo-app.properties index dd22cb05..ebeef4ab 100644 --- a/ansible/roles/fluo/templates/fluo-app.properties +++ b/ansible/roles/fluo/templates/fluo-app.properties @@ -20,7 +20,7 @@ ############################# # NOTE - All properties that have a default are set with it. Uncomment -# a property if you want to use a value different than the default. +# a property if you want to use a value different than the default. # Properties that have no default are uncommented and must be set by # the user. @@ -29,7 +29,7 @@ ## Specifies an observer provider. This should be the name of a class that ## implements org.apache.fluo.api.observer.ObserverProvider. #fluo.observer.provider=com.foo.AppObserverProvider -## Observer jars in this directory are copied to the DFS (specified by fluo.dfs.root) during initialization. +## Observer jars in this directory are copied to the DFS (specified by fluo.dfs.root) during initialization. ## If this property is set, fluo.observer.jars.url should not be set. #fluo.observer.init.dir=/path/to/observer/jars/ ## Observer jars are retreived from this URL. If set, fluo.observer.init.dir should not be set. diff --git a/ansible/roles/fluo/templates/fluo-conn.properties b/ansible/roles/fluo/templates/fluo-conn.properties index 29c0cb38..bcd67252 100644 --- a/ansible/roles/fluo/templates/fluo-conn.properties +++ b/ansible/roles/fluo/templates/fluo-conn.properties @@ -20,7 +20,7 @@ ############################ # NOTE - All properties that have a default are set with it. Uncomment -# a property if you want to use a value different than the default. +# a property if you want to use a value different than the default. # Properties that have no default are uncommented and must be set by # the user. diff --git a/ansible/roles/fluo/templates/fluo.properties b/ansible/roles/fluo/templates/fluo.properties index 1ff4df3b..d3ae4731 100644 --- a/ansible/roles/fluo/templates/fluo.properties +++ b/ansible/roles/fluo/templates/fluo.properties @@ -20,7 +20,7 @@ ################# # NOTE - All properties that have a default are set with it. Uncomment -# a property if you want to use a value different than the default. +# a property if you want to use a value different than the default. # Properties that have no default are uncommented and must be set by # the user. Most are unset except for fluo.accumulo.classpath which # has a suggested value. diff --git a/ansible/roles/fluo_yarn/templates/fluo-yarn.properties b/ansible/roles/fluo_yarn/templates/fluo-yarn.properties index 2ae8884f..900433b4 100644 --- a/ansible/roles/fluo_yarn/templates/fluo-yarn.properties +++ b/ansible/roles/fluo_yarn/templates/fluo-yarn.properties @@ -20,7 +20,7 @@ ######################## # NOTE - All properties that have a default are set with it. Uncomment -# a property if you want to use a value different than the default. +# a property if you want to use a value different than the default. # Properties that have no default are uncommented and must be set by # the user. diff --git a/ansible/roles/grafana/tasks/main.yml b/ansible/roles/grafana/tasks/main.yml index 35c86d26..e66d2192 100644 --- a/ansible/roles/grafana/tasks/main.yml +++ b/ansible/roles/grafana/tasks/main.yml @@ -16,8 +16,8 @@ # - name: "download grafana rpm" - get_url: - args: + get_url: + args: url: https://grafanarel.s3.amazonaws.com/builds/{{ grafana_rpm }} dest: /tmp/{{ grafana_rpm }} sha256sum: "{{ grafana_sha256 }}" diff --git a/ansible/roles/hadoop/templates/core-site.xml b/ansible/roles/hadoop/templates/core-site.xml index 4f44f2ea..409c1033 100644 --- a/ansible/roles/hadoop/templates/core-site.xml +++ b/ansible/roles/hadoop/templates/core-site.xml @@ -23,7 +23,7 @@ - fs.defaultFS + fs.defaultFS {{ hdfs_root }} diff --git a/ansible/roles/hadoop/templates/hdfs-site.xml b/ansible/roles/hadoop/templates/hdfs-site.xml index f102feac..684d4c41 100644 --- a/ansible/roles/hadoop/templates/hdfs-site.xml +++ b/ansible/roles/hadoop/templates/hdfs-site.xml @@ -18,19 +18,6 @@ limitations under the License. --> - @@ -45,8 +32,8 @@ dfs.datanode.data.dir - {% for dir in worker_data_dirs -%} - {{ dir }}/hadoop/data + {% for dir in worker_data_dirs -%} + {{ dir }}/hadoop/data {%- if not loop.last -%} , {%- endif -%} {%- endfor %} diff --git a/ansible/roles/hadoop/templates/mapred-site.xml b/ansible/roles/hadoop/templates/mapred-site.xml index c6be0cec..7dbb5f6a 100644 --- a/ansible/roles/hadoop/templates/mapred-site.xml +++ b/ansible/roles/hadoop/templates/mapred-site.xml @@ -28,14 +28,14 @@ mapreduce.cluster.temp.dir - {% for dir in worker_data_dirs -%} + {% for dir in worker_data_dirs -%} {{ dir }}/hadoop/mapred/temp {%- if not loop.last -%} , {%- endif -%} {%- endfor %} mapreduce.cluster.local.dir - {% for dir in worker_data_dirs -%} + {% for dir in worker_data_dirs -%} {{ dir }}/hadoop/mapred/local {%- if not loop.last -%} , {%- endif -%} {%- endfor %} diff --git a/ansible/roles/hadoop/templates/yarn-site.xml b/ansible/roles/hadoop/templates/yarn-site.xml index 85033a6b..c0ba8d9c 100644 --- a/ansible/roles/hadoop/templates/yarn-site.xml +++ b/ansible/roles/hadoop/templates/yarn-site.xml @@ -18,19 +18,6 @@ limitations under the License. --> - @@ -41,7 +28,7 @@ yarn.nodemanager.local-dirs - {% for dir in worker_data_dirs -%} + {% for dir in worker_data_dirs -%} {{ dir }}/hadoop/yarn/local {%- if not loop.last -%} , {%- endif -%} {%- endfor %} diff --git a/ansible/roles/influxdb/tasks/main.yml b/ansible/roles/influxdb/tasks/main.yml index ae83a2e5..1f2897df 100644 --- a/ansible/roles/influxdb/tasks/main.yml +++ b/ansible/roles/influxdb/tasks/main.yml @@ -17,7 +17,7 @@ - name: "download influxdb rpm" get_url: - args: + args: url: https://influxdb.s3.amazonaws.com/{{ influxdb_rpm }} dest: /tmp/{{ influxdb_rpm }} sha256sum: "{{ influxdb_sha256 }}" diff --git a/ansible/roles/proxy/tasks/download.yml b/ansible/roles/proxy/tasks/download.yml index 57440709..48a3a7c4 100644 --- a/ansible/roles/proxy/tasks/download.yml +++ b/ansible/roles/proxy/tasks/download.yml @@ -31,4 +31,4 @@ - { urlp: "{{ apache_mirror.stdout }}/zookeeper/zookeeper-{{ zookeeper_version }}", fn: "{{ zookeeper_tarball }}", sum: "{{ zookeeper_sha256 }}" } - { urlp: "{{ apache_mirror.stdout }}/hadoop/common/hadoop-{{ hadoop_version }}", fn: "{{ hadoop_tarball }}", sum: "{{ hadoop_sha256 }}" } - { urlp: "{{ apache_mirror.stdout }}/maven/maven-3/{{ maven_version }}/binaries", fn: "{{ maven_tarball }}", sum: "{{ maven_sha256 }}" } - - { urlp: "https://github.com/github/hub/releases/download/v{{ hub_version }}", fn: "{{ hub_tarball }}", sum: "{{ hub_sha256 }}" } + - { urlp: "https://github.com/github/hub/releases/download/v{{ hub_version }}", fn: "{{ hub_tarball }}", sum: "{{ hub_sha256 }}" } diff --git a/ansible/roles/proxy/tasks/main.yml b/ansible/roles/proxy/tasks/main.yml index 21a7fa52..b7ab1da0 100644 --- a/ansible/roles/proxy/tasks/main.yml +++ b/ansible/roles/proxy/tasks/main.yml @@ -15,7 +15,7 @@ # limitations under the License. # -- name: "ensure cluster user exists and generate ssh key" +- name: "ensure cluster user exists and generate ssh key" user: name={{ cluster_user }} generate_ssh_key=yes ssh_key_bits=4096 state=present become: yes - name: "create tarball directory on proxy" diff --git a/ansible/roles/spark/tasks/start-spark-history.yml b/ansible/roles/spark/tasks/start-spark-history.yml index 0b092b9b..ad136c8c 100644 --- a/ansible/roles/spark/tasks/start-spark-history.yml +++ b/ansible/roles/spark/tasks/start-spark-history.yml @@ -15,7 +15,7 @@ # limitations under the License. # -- name: "ensure spark history directory exists in hdfs" +- name: "ensure spark history directory exists in hdfs" command: "{{ hadoop_home}}/bin/hdfs dfs -mkdir -p /spark/history" register: mk_hist_dir changed_when: mk_hist_dir.rc != 0 diff --git a/ansible/roles/zookeeper/templates/zoo.cfg b/ansible/roles/zookeeper/templates/zoo.cfg index 1c154a68..2eda8f65 100644 --- a/ansible/roles/zookeeper/templates/zoo.cfg +++ b/ansible/roles/zookeeper/templates/zoo.cfg @@ -1,13 +1,13 @@ # The number of milliseconds of each tick tickTime=2000 -# The number of ticks that the initial +# The number of ticks that the initial # synchronization phase can take initLimit=10 -# The number of ticks that can pass between +# The number of ticks that can pass between # sending a request and getting an acknowledgement syncLimit=5 # the directory where the snapshot is stored. -# do not use /tmp for storage, /tmp here is just +# do not use /tmp for storage, /tmp here is just # example sakes. dataDir={{ default_data_dirs[0] }}/zookeeper # the port at which the clients will connect diff --git a/conf/.gitignore b/conf/.gitignore index dc331b4d..0e1a2fcd 100644 --- a/conf/.gitignore +++ b/conf/.gitignore @@ -1,4 +1,4 @@ -muchos.props* -hosts/* -keys -user_data +/muchos.props* +/hosts/* +/keys +/user_data diff --git a/conf/templates/README.md b/conf/templates/README.md index cb15e832..099f0828 100644 --- a/conf/templates/README.md +++ b/conf/templates/README.md @@ -6,31 +6,31 @@ your EC2 hosts. For example, cluster templates may be ideal for use cases that require distinct, per-host launch configurations, and for use cases that require hosts to have persistent, EBS-backed data volumes (rather than ephemeral volumes, the muchos default for EC2 -clusters) +clusters). If you are already familiar with muchos and with the basics of EC2 launch requests, then creating your own launch templates will be simple and straightforward. Please follow the guidance provided here to ensure compatibility between -your custom templates and muchos automation +your custom templates and muchos automation. ## Configuration ### Select a cluster template in *muchos.props* -``` + +```ini [ec2] ... cluster_template = example ... ``` + The configured value must match the name of a subdirectory under `conf/templates` -``` -~$ ls -1a fluo-muchos/conf/templates/example -. -.. +```sh +~$ ls -1 fluo-muchos/conf/templates/example devices metrics.json namenode.json @@ -38,21 +38,23 @@ resourcemanager.json zookeeper.json worker.json ``` + The subdirectory will contain one or more user-defined EC2 launch -templates (*\*.json*) for your various host types, and it will -include a *devices* file specifying the desired mount points for all +templates `*.json` for your various host service types, and it will +include a `devices` file specifying the desired mount points for all data volumes (excluding root volumes, as they are mounted -automatically) +automatically). ### Defining EC2 launch templates and device mounts for your hosts -#### Launch Templates: *{service-name}.json* files +#### Launch Templates: `{service-name}.json` files Each JSON file represents a standard EC2 launch request, and each file name must match one of the predefined muchos service names, as defined in the **nodes** section of *muchos.props*. E.g., -``` -... + +```ini +... [nodes] leader1 = namenode,resourcemanager,accumulomaster leader2 = metrics,zookeeper @@ -61,129 +63,131 @@ worker2 = worker worker3 = worker worker4 = worker ``` + In template mode, the first service listed for a given host denotes the template to be selected for its launch. Based on the example given above: -* **leader1** selects **namenode.json** -* **leader2** selects **metrics.json** -* **worker1** selects **worker.json** +* **leader1** selects `namenode.json` +* **leader2** selects `metrics.json` +* **worker1** selects `worker.json` * and so on... - -For example, *namenode.json* might be defined as follows... -``` + +For example, `namenode.json` might be defined as follows... + +```json { - "KeyName": "${key_name}", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", # Here, /dev/sda1 denotes the root volume - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 40, - "VolumeType": "gp2" - } - }, - { - "DeviceName": "/dev/sdf", # Here, /dev/sdf (a.k.a "/dev/xvdf") denotes - "Ebs": { # our single EBS-backed data volume - "DeleteOnTermination": true, - "VolumeSize": 500, - "VolumeType": "gp2" - } - } - ], - "ImageId": "${aws_ami}", - "InstanceType": "m4.4xlarge", - "NetworkInterfaces": [ - { - "DeviceIndex": 0, - "AssociatePublicIpAddress": ${associate_public_ip}, - "Groups": [ - "${security_group_id}" - ] - } - ], - "EbsOptimized": true, - "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" + "KeyName": "${key_name}", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", # Here, /dev/sda1 denotes the root volume + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 40, + "VolumeType": "gp2" + } + }, + { + "DeviceName": "/dev/sdf", # Here, /dev/sdf (a.k.a "/dev/xvdf") denotes + "Ebs": { # our single EBS-backed data volume + "DeleteOnTermination": true, + "VolumeSize": 500, + "VolumeType": "gp2" + } + } + ], + "ImageId": "${aws_ami}", + "InstanceType": "m4.4xlarge", + "NetworkInterfaces": [ + { + "DeviceIndex": 0, + "AssociatePublicIpAddress": "${associate_public_ip}", + "Groups": [ + "${security_group_id}" + ] + } + ], + "EbsOptimized": true, + "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" } ``` -*Property Placeholders* + +#### Property Placeholders The `${property name}` placeholders demonstrated above are optional and are intended to simplify template creation and reduce maintenance burden. -They allow any matching properties from the **ec2** section of *muchos.props* +They allow any matching properties from the **ec2** section of `muchos.props` to be interpolated automatically. If needed, you may also define your own custom properties and have them be injected automatically by simply adding them to the **ec2** section of -*muchos.props* and to your templates +`muchos.props` and to your templates -#### Device Mounts: *devices* file +#### Device Mounts: `devices` file -The *devices* file contains the user-defined mapping of storage +The `devices` file contains the user-defined mapping of storage devices and mount points for all data (i.e., non-root) volumes in your cluster. Muchos Ansible scripts leverage this information to prepare all data volumes during the cluster `setup` phase. -Two (and only two) device mappings should exist within *devices*: +Two (and only two) device mappings should exist within `devices`: * One map to represent your **worker** device mounts, and * One map to represent the device mounts on all other hosts, i.e., the **default** map - -For example, the *devices* file below specifies 4 mount points for all + +For example, the `devices` file below specifies 4 mount points for all `worker` instance types, and specifies 1 mount point for all other hosts via the `default` map. -``` + +```json { - "default": { - "mounts": [ - "/data0" # For non-workers, mount the /data0 directory - ], # on /dev/xvdf (a.k.a "/dev/sdf") - "devices": [ - "/dev/xvdf" - ] + "default": { + "mounts": [ + "/data0" # For non-workers, mount the /data0 directory + ], # on /dev/xvdf (a.k.a "/dev/sdf") + "devices": [ + "/dev/xvdf" + ] }, "worker": { - "mounts": [ - "/data0", - "/data1", - "/data2", - "/data3" # For workers, mount the /data0 directory on - ], # /dev/xvdf (a.k.a "/dev/sdf"), mount /data1 on - "devices": [ # /dev/xvdg (a.k.a "/dev/sdg"), and so on... - "/dev/xvdf", - "/dev/xvdg", - "/dev/xvdh", - "/dev/xvdi" - ] + "mounts": [ + "/data0", + "/data1", + "/data2", + "/data3" # For workers, mount the /data0 directory on + ], # /dev/xvdf (a.k.a "/dev/sdf"), mount /data1 on + "devices": [ # /dev/xvdg (a.k.a "/dev/sdg"), and so on... + "/dev/xvdf", + "/dev/xvdg", + "/dev/xvdh", + "/dev/xvdi" + ] } } ``` + Naturally, you should take care to ensure that your **BlockDeviceMappings** also align to *default* vs *worker* node type semantics. As you explore the example files, you should observe the implicit link betweeen -a data volume denoted by *BlockDeviceMappings\[N].DeviceName* and its -respective device map entry in the *devices* file. +a data volume denoted by `BlockDeviceMappings[N].DeviceName` and its +respective device map entry in the `devices` file. * **Note**: While *DeviceName* mount profiles should not vary among your default (non-worker) nodes, other attributes within *BlockDeviceMappings*, such as *DeleteOnTermination*, *VolumeSize*, *etc*, may vary as needed - * **Note**: Be aware that the device names used by AWS EC2 launch requests may differ from the actual names assigned by the EC2 block device driver, which can be confusing. Please read [this](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html) and [also this](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html) for additional information and guidance, if needed - * **Note**: Root EBS volumes may be configured as desired in your JSON launch templates, but only non-root, "data" storage devices should be - specified in *devices*, as root devices are mounted automatically - - + specified in `devices`, as root devices are mounted automatically + ## Beyond the Launch Phase: *Setup*, *Terminate*, *Etc* Aside from the configuration differences described above, which impact the `launch` phase, all other muchos operations behave the same in -template mode as in default mode - +template mode as in default mode. + diff --git a/conf/templates/example/accumulomaster.json b/conf/templates/example/accumulomaster.json index 263c1920..5569994c 100644 --- a/conf/templates/example/accumulomaster.json +++ b/conf/templates/example/accumulomaster.json @@ -1,34 +1,34 @@ { - "KeyName": "${key_name}", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 40, - "VolumeType": "gp2" - } - }, - { - "DeviceName": "/dev/sdf", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 500, - "VolumeType": "gp2" - } - } - ], - "ImageId": "${aws_ami}", - "InstanceType": "m4.2xlarge", - "NetworkInterfaces": [ - { - "DeviceIndex": 0, - "AssociatePublicIpAddress": ${associate_public_ip}, - "Groups": [ - "${security_group_id}" - ] - } - ], - "EbsOptimized": true, - "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" + "KeyName": "${key_name}", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 40, + "VolumeType": "gp2" + } + }, + { + "DeviceName": "/dev/sdf", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 500, + "VolumeType": "gp2" + } + } + ], + "ImageId": "${aws_ami}", + "InstanceType": "m4.2xlarge", + "NetworkInterfaces": [ + { + "DeviceIndex": 0, + "AssociatePublicIpAddress": "${associate_public_ip}", + "Groups": [ + "${security_group_id}" + ] + } + ], + "EbsOptimized": true, + "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" } diff --git a/conf/templates/example/client.json b/conf/templates/example/client.json index c5d4de1b..5569994c 100644 --- a/conf/templates/example/client.json +++ b/conf/templates/example/client.json @@ -1,34 +1,34 @@ { - "KeyName": "${key_name}", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 40, - "VolumeType": "gp2" - } - }, - { - "DeviceName": "/dev/sdf", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 500, - "VolumeType": "gp2" - } - } - ], - "ImageId": "${aws_ami}", - "InstanceType": "m4.2xlarge", - "NetworkInterfaces": [ - { - "DeviceIndex": 0, - "AssociatePublicIpAddress": ${associate_public_ip}, - "Groups": [ - "${security_group_id}" - ] - } - ], - "EbsOptimized": true, - "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" + "KeyName": "${key_name}", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 40, + "VolumeType": "gp2" + } + }, + { + "DeviceName": "/dev/sdf", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 500, + "VolumeType": "gp2" + } + } + ], + "ImageId": "${aws_ami}", + "InstanceType": "m4.2xlarge", + "NetworkInterfaces": [ + { + "DeviceIndex": 0, + "AssociatePublicIpAddress": "${associate_public_ip}", + "Groups": [ + "${security_group_id}" + ] + } + ], + "EbsOptimized": true, + "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" } diff --git a/conf/templates/example/devices b/conf/templates/example/devices index f338f9f4..5d490872 100644 --- a/conf/templates/example/devices +++ b/conf/templates/example/devices @@ -21,4 +21,4 @@ "/dev/xvdi" ] } -} \ No newline at end of file +} diff --git a/conf/templates/example/metrics.json b/conf/templates/example/metrics.json index c5d4de1b..5569994c 100644 --- a/conf/templates/example/metrics.json +++ b/conf/templates/example/metrics.json @@ -1,34 +1,34 @@ { - "KeyName": "${key_name}", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 40, - "VolumeType": "gp2" - } - }, - { - "DeviceName": "/dev/sdf", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 500, - "VolumeType": "gp2" - } - } - ], - "ImageId": "${aws_ami}", - "InstanceType": "m4.2xlarge", - "NetworkInterfaces": [ - { - "DeviceIndex": 0, - "AssociatePublicIpAddress": ${associate_public_ip}, - "Groups": [ - "${security_group_id}" - ] - } - ], - "EbsOptimized": true, - "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" + "KeyName": "${key_name}", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 40, + "VolumeType": "gp2" + } + }, + { + "DeviceName": "/dev/sdf", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 500, + "VolumeType": "gp2" + } + } + ], + "ImageId": "${aws_ami}", + "InstanceType": "m4.2xlarge", + "NetworkInterfaces": [ + { + "DeviceIndex": 0, + "AssociatePublicIpAddress": "${associate_public_ip}", + "Groups": [ + "${security_group_id}" + ] + } + ], + "EbsOptimized": true, + "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" } diff --git a/conf/templates/example/namenode.json b/conf/templates/example/namenode.json index b2817826..1cf398a8 100644 --- a/conf/templates/example/namenode.json +++ b/conf/templates/example/namenode.json @@ -1,34 +1,34 @@ { - "KeyName": "${key_name}", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 40, - "VolumeType": "gp2" - } - }, - { - "DeviceName": "/dev/sdf", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 500, - "VolumeType": "gp2" - } - } - ], - "ImageId": "${aws_ami}", - "InstanceType": "m4.4xlarge", - "NetworkInterfaces": [ - { - "DeviceIndex": 0, - "AssociatePublicIpAddress": ${associate_public_ip}, - "Groups": [ - "${security_group_id}" - ] - } - ], - "EbsOptimized": true, - "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" + "KeyName": "${key_name}", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 40, + "VolumeType": "gp2" + } + }, + { + "DeviceName": "/dev/sdf", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 500, + "VolumeType": "gp2" + } + } + ], + "ImageId": "${aws_ami}", + "InstanceType": "m4.4xlarge", + "NetworkInterfaces": [ + { + "DeviceIndex": 0, + "AssociatePublicIpAddress": "${associate_public_ip}", + "Groups": [ + "${security_group_id}" + ] + } + ], + "EbsOptimized": true, + "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" } diff --git a/conf/templates/example/resourcemanager.json b/conf/templates/example/resourcemanager.json index a406feb0..5569994c 100644 --- a/conf/templates/example/resourcemanager.json +++ b/conf/templates/example/resourcemanager.json @@ -1,34 +1,34 @@ { - "KeyName": "${key_name}", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 40, - "VolumeType": "gp2" - } - }, - { - "DeviceName": "/dev/sdf", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 500, - "VolumeType": "gp2" - } - } - ], - "ImageId": "${aws_ami}", - "InstanceType": "m4.2xlarge", - "NetworkInterfaces": [ - { - "DeviceIndex": 0, - "AssociatePublicIpAddress": ${associate_public_ip}, - "Groups": [ - "${security_group_id}" - ] - } - ], - "EbsOptimized": true, - "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" + "KeyName": "${key_name}", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 40, + "VolumeType": "gp2" + } + }, + { + "DeviceName": "/dev/sdf", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 500, + "VolumeType": "gp2" + } + } + ], + "ImageId": "${aws_ami}", + "InstanceType": "m4.2xlarge", + "NetworkInterfaces": [ + { + "DeviceIndex": 0, + "AssociatePublicIpAddress": "${associate_public_ip}", + "Groups": [ + "${security_group_id}" + ] + } + ], + "EbsOptimized": true, + "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" } diff --git a/conf/templates/example/worker.json b/conf/templates/example/worker.json index ae693eee..ebd581e5 100644 --- a/conf/templates/example/worker.json +++ b/conf/templates/example/worker.json @@ -1,58 +1,58 @@ { - "KeyName": "${key_name}", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 40, - "VolumeType": "gp2" - } - }, - { - "DeviceName": "/dev/sdf", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 250, - "VolumeType": "gp2" - } - }, - { - "DeviceName": "/dev/sdg", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 250, - "VolumeType": "gp2" - } - }, - { - "DeviceName": "/dev/sdh", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 250, - "VolumeType": "gp2" - } - }, - { - "DeviceName": "/dev/sdi", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 250, - "VolumeType": "gp2" - } - } - ], - "ImageId": "${aws_ami}", - "InstanceType": "c4.4xlarge", - "NetworkInterfaces": [ - { - "DeviceIndex": 0, - "AssociatePublicIpAddress": ${associate_public_ip}, - "Groups": [ - "${security_group_id}" - ] - } - ], - "EbsOptimized": true, - "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" + "KeyName": "${key_name}", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 40, + "VolumeType": "gp2" + } + }, + { + "DeviceName": "/dev/sdf", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 250, + "VolumeType": "gp2" + } + }, + { + "DeviceName": "/dev/sdg", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 250, + "VolumeType": "gp2" + } + }, + { + "DeviceName": "/dev/sdh", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 250, + "VolumeType": "gp2" + } + }, + { + "DeviceName": "/dev/sdi", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 250, + "VolumeType": "gp2" + } + } + ], + "ImageId": "${aws_ami}", + "InstanceType": "c4.4xlarge", + "NetworkInterfaces": [ + { + "DeviceIndex": 0, + "AssociatePublicIpAddress": "${associate_public_ip}", + "Groups": [ + "${security_group_id}" + ] + } + ], + "EbsOptimized": true, + "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" } diff --git a/conf/templates/example/zookeeper.json b/conf/templates/example/zookeeper.json index 16ae67ec..7add19e0 100644 --- a/conf/templates/example/zookeeper.json +++ b/conf/templates/example/zookeeper.json @@ -1,34 +1,34 @@ { - "KeyName": "${key_name}", - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 40, - "VolumeType": "gp2" - } - }, - { - "DeviceName": "/dev/sdf", - "Ebs": { - "DeleteOnTermination": true, - "VolumeSize": 500, - "VolumeType": "gp2" - } - } - ], - "ImageId": "${aws_ami}", - "InstanceType": "m4.xlarge", - "NetworkInterfaces": [ - { - "DeviceIndex": 0, - "AssociatePublicIpAddress": ${associate_public_ip}, - "Groups": [ - "${security_group_id}" - ] - } - ], - "EbsOptimized": true, - "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" + "KeyName": "${key_name}", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 40, + "VolumeType": "gp2" + } + }, + { + "DeviceName": "/dev/sdf", + "Ebs": { + "DeleteOnTermination": true, + "VolumeSize": 500, + "VolumeType": "gp2" + } + } + ], + "ImageId": "${aws_ami}", + "InstanceType": "m4.xlarge", + "NetworkInterfaces": [ + { + "DeviceIndex": 0, + "AssociatePublicIpAddress": "${associate_public_ip}", + "Groups": [ + "${security_group_id}" + ] + } + ], + "EbsOptimized": true, + "InstanceInitiatedShutdownBehavior": "${shutdown_behavior}" } diff --git a/conf/templates/user/.gitignore b/conf/templates/user/.gitignore index 72e8ffc0..33662f55 100644 --- a/conf/templates/user/.gitignore +++ b/conf/templates/user/.gitignore @@ -1 +1 @@ -* +/* diff --git a/conf/upload/.gitignore b/conf/upload/.gitignore index 9db4ee63..fa80b408 100644 --- a/conf/upload/.gitignore +++ b/conf/upload/.gitignore @@ -1,2 +1,2 @@ -*.tar.gz -*.tgz +/*.tar.gz +/*.tgz diff --git a/contrib/add-license-headers.md b/contrib/add-license-headers.md index 723ab64a..bffc6983 100644 --- a/contrib/add-license-headers.md +++ b/contrib/add-license-headers.md @@ -1,16 +1,15 @@ ## Adding licence header to Muchos files -Using [license-maven-plugin][license-plugin] we can generate a consistent license +Using [license-maven-plugin][license-plugin] we can generate a consistent license header for the project with the following command. - - * Temporally convert the project to a maven project. Make sure not to commit the new `pom.xml` file. * Via command line, run the following command to generate the license header. -``` + +```sh mvn com.mycila:license-maven-plugin:3.0:format -Dlicense.header=contrib/license-header.txt ``` * Check the output for warnings and manually add the appropriate license to those files. -[license-plugin]: https://code.mycila.com/license-maven-plugin/ \ No newline at end of file +[license-plugin]: https://code.mycila.com/license-maven-plugin/ diff --git a/contrib/license-header.txt b/contrib/license-header.txt index 9f4ff314..8853bce3 100644 --- a/contrib/license-header.txt +++ b/contrib/license-header.txt @@ -11,4 +11,4 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file +limitations under the License. diff --git a/contrib/muchos-logo-dark.svg b/contrib/muchos-logo-dark.svg index 5e978dcb..d82dcda8 100644 --- a/contrib/muchos-logo-dark.svg +++ b/contrib/muchos-logo-dark.svg @@ -15,10 +15,9 @@ See the License for the specific language governing permissions and limitations under the License. --> - - diff --git a/contrib/muchos-logo.svg b/contrib/muchos-logo.svg index c5a4418e..320887f5 100644 --- a/contrib/muchos-logo.svg +++ b/contrib/muchos-logo.svg @@ -15,9 +15,9 @@ See the License for the specific language governing permissions and limitations under the License. --> - diff --git a/contrib/pom.xml b/contrib/pom.xml new file mode 100644 index 00000000..48f9e2c5 --- /dev/null +++ b/contrib/pom.xml @@ -0,0 +1,28 @@ + + + + 4.0.0 + org.apache.fluo + muchos + 0.0.1-SNAPSHOT + pom + Apache Fluo Muchos + This is a dummy POM for use with contrib/add-license-headers.md + diff --git a/lib/.gitignore b/lib/.gitignore index 70e362f7..1f67f4de 100644 --- a/lib/.gitignore +++ b/lib/.gitignore @@ -1 +1 @@ -boto* +/boto*