diff --git a/ansible/MANUAL_STEPS.md b/ansible/MANUAL_STEPS.md new file mode 100644 index 000000000..f2fd8ad32 --- /dev/null +++ b/ansible/MANUAL_STEPS.md @@ -0,0 +1,18 @@ +# Manual steps required to run ansible on machines + +## macOS +1. Update Sudoers file: + +this requires `NOPASSWD` to be added to the sudoers file to enable elevation + +`sudo visudo` +and change: +`%admin ALL = (ALL) ALL` +to +`%admin ALL = (ALL) NOPASSWD:ALL` + +2. Allow ssh access + +```bash +sudo systemsetup -setremotelogin on +``` diff --git a/ansible/roles/baselayout/handlers/main.yml b/ansible/roles/baselayout/handlers/main.yml index 22876406e..af454b351 100644 --- a/ansible/roles/baselayout/handlers/main.yml +++ b/ansible/roles/baselayout/handlers/main.yml @@ -5,4 +5,5 @@ # - name: restart sshd + when: not os|startswith("macos") service: name="{{ sshd_service_name }}" state=restarted diff --git a/ansible/roles/baselayout/tasks/main.yml b/ansible/roles/baselayout/tasks/main.yml index 8585bb9f7..f1fe9fe87 100644 --- a/ansible/roles/baselayout/tasks/main.yml +++ b/ansible/roles/baselayout/tasks/main.yml @@ -17,6 +17,7 @@ - name: set hostname when: not os|startswith("smartos") and not os|startswith("zos") + and not os|startswith("macos") hostname: name="{{ safe_hostname }}" - name: disable joyent smartconnect @@ -60,7 +61,19 @@ state: absent - name: install packages - when: not os|startswith("zos") + when: not os|startswith("zos") and not os|startswith("macos") + package: name="{{ package }}" state=present + loop_control: + loop_var: package + with_items: + # ansible doesn't like empty lists + - "{{ packages[os]|default('[]') }}" + - "{{ packages[os|stripversion]|default('[]') }}" + - "{{ common_packages|default('[]') }}" + +- name: install packages (macos) + when: os|startswith("macos") + become_user: administrator package: name="{{ package }}" state=present loop_control: loop_var: package @@ -71,6 +84,20 @@ - "{{ packages[os|stripversion]|default('[]') }}" - "{{ common_packages|default('[]') }}" +- name: Check whether /etc/paths contains "/usr/localopt/ccache/libexec" (macos) + when: os|startswith("macos") + command: grep -Fxq "/usr/localopt/ccache/libexec" /etc/paths + register: ccache_mac + check_mode: no + ignore_errors: yes + changed_when: no + +- name: add ccache to the path (macos) + when: ccache_mac.rc == 1 + lineinfile: dest=/etc/paths + insertbefore=BOF + line='/usr/localopt/ccache/libexec' + - name: ubuntu1404 | update package alternatives when: os == "ubuntu1404" alternatives: link=/usr/bin/{{ gcc }} name={{ gcc }} path=/usr/bin/{{ gcc }}-4.9 diff --git a/ansible/roles/baselayout/vars/main.yml b/ansible/roles/baselayout/vars/main.yml index 169e76fe7..a15ada189 100644 --- a/ansible/roles/baselayout/vars/main.yml +++ b/ansible/roles/baselayout/vars/main.yml @@ -27,7 +27,7 @@ ntp_service: { } common_packages: [ - 'automake,bash,libtool,sudo', + 'automake,bash,libtool', ] # you can either add os family or os to this list (see smartos) @@ -35,45 +35,46 @@ common_packages: [ packages: { centos6_x64: ['centos-release-scl'], # only available on x86_64, 32-bit is from https://copr.fedorainfracloud.org/coprs/mlampe centos6: [ - 'ccache,git2u,gcc-c++,devtoolset-6', # even need gcc on centos6 so ccache has symlinks + 'ccache,git2u,gcc-c++,devtoolset-6,sudo', # even need gcc on centos6 so ccache has symlinks ], centos7_arm64: ['git'], # git2u not available for aarch64 (yet) - centos7_x64: ['git2u','centos-release-scl'], # centos-release-scl is required to enable SCLo + centos7_x64: ['git2u','centos-release-scl',], # centos-release-scl is required to enable SCLo # but we do it manually in partials/repo/centos7.yml for arm64 centos7: [ - 'ccache,gcc-c++,devtoolset-6', + 'ccache,gcc-c++,devtoolset-6,sudo', ], debian7: [ - 'gcc-4.8,g++-4.8', + 'gcc-4.8,g++-4.8,sudo', ], debian8: [ - 'ccache,git,gcc-4.9,g++-4.9,libfontconfig1,binutils-2.26', + 'ccache,git,gcc-4.9,g++-4.9,libfontconfig1,binutils-2.26,sudo', ], debian9: [ - 'gcc-6,g++-6,ccache,git,curl,libfontconfig1,apt-transport-https,ca-certificates', + 'gcc-6,g++-6,ccache,git,curl,libfontconfig1,apt-transport-https,ca-certificates,sudo', ], fedora: [ - 'bzip2,ccache,gcc-c++,git,fontconfig', + 'bzip2,ccache,gcc-c++,git,fontconfig,sudo', ], freebsd: [ - 'ccache,git,gmake' + 'ccache,git,gmake,sudo' ], rhel72: [ - 'gcc-c++', + 'gcc-c++,sudo', ], smartos: [ 'gccmakedep', 'git', 'gmake', - 'xz' + 'xz', + 'sudo', ], smartos14: [ @@ -96,7 +97,7 @@ packages: { ], ubuntu: [ - 'ccache,g++,gcc,git,libfontconfig1', + 'ccache,g++,gcc,git,libfontconfig1,sudo', ], ubuntu1404: [ diff --git a/ansible/roles/java-base/tasks/main.yml b/ansible/roles/java-base/tasks/main.yml index 9c992ab96..bbd41b86c 100644 --- a/ansible/roles/java-base/tasks/main.yml +++ b/ansible/roles/java-base/tasks/main.yml @@ -48,12 +48,18 @@ # if this fails you want to check in vars/main.yml and add package name # as appropriate -- try to use generic os family if available. + +- name: Check if java is already installed + raw: java -version + register: java + ignore_errors: yes + - name: install java - when: not os|startswith("zos") and arch != "ppc64" and not inventory_hostname|regex_search('-arm(v6l|v7l|64)_pi') + when: java.rc > 0 and not os|startswith("zos") and arch != "ppc64" and not inventory_hostname|regex_search('-arm(v6l|v7l|64)_pi') package: name="{{ java_package_name }}" state=present - name: install webupd8 oracle java 8 extras - when: os in ("ubuntu1204", "ubuntu1404") and arch != "ppc64" + when: java.rc > 0 and os in ("ubuntu1204", "ubuntu1404") and arch != "ppc64" package: name="{{item}}" state=present with_items: - ca-certificates diff --git a/ansible/roles/java-base/vars/main.yml b/ansible/roles/java-base/vars/main.yml index 32a30fe24..c3c0a588e 100644 --- a/ansible/roles/java-base/vars/main.yml +++ b/ansible/roles/java-base/vars/main.yml @@ -12,6 +12,7 @@ packages: { 'debian9': 'openjdk-8-jre-headless', 'fedora': 'java-1.8.0-openjdk-headless', 'freebsd': 'openjdk8-jre', + 'macos': 'adoptopenjdk-openjdk8' 'rhel72': 'java-1.8.0-openjdk', 'smartos': 'openjdk8', 'ubuntu': 'openjdk-8-jre-headless', diff --git a/ansible/roles/jenkins-worker/tasks/main.yml b/ansible/roles/jenkins-worker/tasks/main.yml index d25b1634a..a65a548be 100644 --- a/ansible/roles/jenkins-worker/tasks/main.yml +++ b/ansible/roles/jenkins-worker/tasks/main.yml @@ -67,11 +67,11 @@ line: ::1 localhost.localdomain localhost - name: create NODE_TEST_DIR directory - file: path=/home/{{ server_user }}/tmp state=directory + file: path="{{ home }}/{{ server_user }}/tmp" state=directory - name: set NODE_TEST_DIR permission and owner file: - path: /home/{{ server_user }}/tmp + path: "{{ home }}/{{ server_user }}/tmp" owner: "{{ server_user }}" group: "{{ server_user }}" mode: 0755 @@ -124,7 +124,7 @@ timeout: 60 force: yes -# temporary until we get the righ cert bundles +# temporary until we get the right cert bundles - name: download slave.jar -zos when: os|startswith("zos") get_url: @@ -176,6 +176,14 @@ dest: "/etc/sysconfig/jenkins" mode: "0644" +# TODO - Should this run on every machine? +- name: copy start.sh to {{ home }}/{{ server_user }}/start.sh + when: os|startswith("macos") + template: + src: "start.j2" + dest: "{{ home }}/{{ server_user }}/start.sh" + mode: "0755" + - name: import manifest to svcadm when: os|startswith("smartos") raw: "svccfg import {{ jenkins.dest }}" @@ -219,9 +227,17 @@ command: "chown -R {{ server_user }} {{ home }}/{{ server_user }}/gyp" - name: enable jenkins at startup - general - when: not os|startswith("zos") + when: not os|startswith("zos") and not os|startswith("macos") service: name=jenkins state=started enabled=yes +- name: Load org.nodejs.osx.jenkins.plist into launchctl + when: os|startswith("macos") + command: launchctl load /Library/LaunchDaemons/org.nodejs.osx.jenkins.plist + +- name: Start org.nodejs.osx.jenkins.plist + when: os|startswith("macos") + command: launchctl start org.nodejs.osx.jenkins + # lineinfile does not work on zos due to character conversions # the inserted line ends up in the wrong character set. We # tried a few variations to work around this without diff --git a/ansible/roles/jenkins-worker/tasks/partials/tap2junit/macos.yml b/ansible/roles/jenkins-worker/tasks/partials/tap2junit/macos.yml new file mode 100644 index 000000000..efce67051 --- /dev/null +++ b/ansible/roles/jenkins-worker/tasks/partials/tap2junit/macos.yml @@ -0,0 +1,10 @@ +--- + +# +# macos: python2.7 +# + +- name: install pip + pip: + name: tap2junit + executable: /usr/local/bin/pip2 diff --git a/ansible/roles/jenkins-worker/templates/org.nodejs.osx.jenkins.plist b/ansible/roles/jenkins-worker/templates/org.nodejs.osx.jenkins.plist new file mode 100644 index 000000000..7b3c2c72f --- /dev/null +++ b/ansible/roles/jenkins-worker/templates/org.nodejs.osx.jenkins.plist @@ -0,0 +1,29 @@ + + + + + Label + org.nodejs.osx.jenkins + + UserName + iojs + + WorkingDirectory + /Users/iojs + + Program + /Users/iojs/start.sh + + RunAtLoad + + + KeepAlive + + + StandardErrorPath + /Users/iojs/jenkins_err.log + + StandardOutPath + /Users/iojs/jenkins.log + + diff --git a/ansible/roles/jenkins-worker/templates/start.j2 b/ansible/roles/jenkins-worker/templates/start.j2 index eeef510c3..1e0ede271 100644 --- a/ansible/roles/jenkins-worker/templates/start.j2 +++ b/ansible/roles/jenkins-worker/templates/start.j2 @@ -1,10 +1,5 @@ -#!/NODEJS2/bin/bash - -su -s - iojs <<'EOF' - -export PATH=/NODEJS/bin:$PATH +#!/bin/bash export HOME={{ home }}/{{ server_user }} -export NODE_COMMON_PIPE="$HOME/test.pipe" export NODE_TEST_DIR="$HOME/tmp" export JOBS="{{ jobs_env }}" export OSTYPE=zos diff --git a/ansible/roles/jenkins-worker/templates/zos_start.j2 b/ansible/roles/jenkins-worker/templates/zos_start.j2 new file mode 100644 index 000000000..79374f7fb --- /dev/null +++ b/ansible/roles/jenkins-worker/templates/zos_start.j2 @@ -0,0 +1,26 @@ +#!/NODEJS2/bin/bash + +su -s - iojs <<'EOF' + +export PATH=/NODEJS/bin:$PATH +export HOME={{ home }}/{{ server_user }} +export NODE_COMMON_PIPE="$HOME/test.pipe" +export NODE_TEST_DIR="$HOME/tmp" +export JOBS="{{ server_jobs | default(ansible_processor_vcpus) }}" +export OSTYPE=zos + +export _BPXK_AUTOCVT=ON +export _CEE_RUNOPTS="FILETAG(AUTOCVT,AUTOTAG) POSIX(ON)" +export _TAG_REDIR_ERR=txt +export _TAG_REDIR_IN=txt +export _TAG_REDIR_OUT=txt + +export CC=/bin/xlc +export LINK=/bin/xlc +export CFLAGS="-q64" +export LDFLAGS="-q64" + +{{ java_path[os] }} -Dfile.encoding=ISO8859_1 -Xmx{{ server_ram|default('128m') }} \ + -jar {{ home }}/{{ server_user }}/slave.jar -secret {{ secret }} \ + -jnlpUrl {{ jenkins_url }}/computer/{{ inventory_hostname }}/slave-agent.jnlp >{{ home }}/{{ server_user }}/jenkins.log 2>&1 & +EOF diff --git a/ansible/roles/jenkins-worker/vars/main.yml b/ansible/roles/jenkins-worker/vars/main.yml index e31bc3a00..6ea68bbce 100644 --- a/ansible/roles/jenkins-worker/vars/main.yml +++ b/ansible/roles/jenkins-worker/vars/main.yml @@ -9,11 +9,12 @@ init: { centos6: 'centos6', debian: ['debian7', 'ubuntu1204'], freebsd: 'freebsd', + macos: 'macos', rhel72: 'rhel72', systemd: ['centos7', 'debian8', 'debian9', 'fedora', 'ubuntu1604', 'ubuntu1610', 'ubuntu1710'], svc: 'smartos', upstart: ['ubuntu12', 'ubuntu1404'], - start: 'zos' + zos_start: 'zos' } jenkins_init: { @@ -37,6 +38,11 @@ jenkins_init: { src: 'freebsd.initd.j2', mode: '0755' }, + macos: { + dest: '/Library/LaunchDaemons/org.nodejs.osx.jenkins.plist', + src: 'org.nodejs.osx.jenkins.plist', + mode: '0755' + }, openrc: { dest: '/etc/init.d/jenkins', src: 'openrc.initd.j2', @@ -59,9 +65,9 @@ jenkins_init: { dest: '/etc/init/jenkins.conf', src: 'upstart.j2' }, - start: { + zos_start: { dest: '{{ home }}/{{ server_user }}/start.sh', - src: 'start.j2' + src: 'zos_start.j2' }, } diff --git a/ansible/roles/package-upgrade/files/core-dumps.plist b/ansible/roles/package-upgrade/files/core-dumps.plist new file mode 100644 index 000000000..bca8cb8f3 --- /dev/null +++ b/ansible/roles/package-upgrade/files/core-dumps.plist @@ -0,0 +1,15 @@ + + + + Label + sysctl + ProgramArguments + + /usr/sbin/sysctl + -w + kern.corefile=core.%P + + RunAtLoad + + + diff --git a/ansible/roles/package-upgrade/files/install-homebrew.sh b/ansible/roles/package-upgrade/files/install-homebrew.sh new file mode 100755 index 000000000..b1964b318 --- /dev/null +++ b/ansible/roles/package-upgrade/files/install-homebrew.sh @@ -0,0 +1,2 @@ +#!/bin/bash +yes | /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" diff --git a/ansible/roles/package-upgrade/files/install-xcode.sh b/ansible/roles/package-upgrade/files/install-xcode.sh new file mode 100755 index 000000000..d29dfe5c6 --- /dev/null +++ b/ansible/roles/package-upgrade/files/install-xcode.sh @@ -0,0 +1,44 @@ +#!/bin/bash +osx_vers=$(sw_vers -productVersion | awk -F "." '{print $2}') +cmd_line_tools_temp_file="/tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress" + +# Installing the latest Xcode command line tools on 10.9.x or higher + +if [[ "$osx_vers" -ge 9 ]]; then +touch "$cmd_line_tools_temp_file"; +PROD=$(softwareupdate -l | + grep "\*.*Command Line" | + head -n 1 | awk -F"*" '{print $2}' | + sed -e 's/^ *//' | +tr -d '\n') +softwareupdate -i "$PROD"; +fi + +# Installing the latest Xcode command line tools on 10.7.x and 10.8.x + +# on 10.7/10.8, instead of using the software update feed, the command line tools are downloaded +# instead from public download URLs, which can be found in the dvtdownloadableindex: +# https://devimages.apple.com.edgekey.net/downloads/xcode/simulators/index-3905972D-B609-49CE-8D06-51ADC78E07BC.dvtdownloadableindex + +if [[ "$osx_vers" -eq 7 ]] || [[ "$osx_vers" -eq 8 ]]; then + + if [[ "$osx_vers" -eq 7 ]]; then + DMGURL=http://devimages.apple.com/downloads/xcode/command_line_tools_for_xcode_os_x_lion_april_2013.dmg + fi + + if [[ "$osx_vers" -eq 8 ]]; then + DMGURL=http://devimages.apple.com/downloads/xcode/command_line_tools_for_osx_mountain_lion_april_2014.dmg + fi + + TOOLS=cltools.dmg + curl "$DMGURL" -o "$TOOLS" + TMPMOUNT=`/usr/bin/mktemp -d /tmp/clitools.XXXX` + hdiutil attach "$TOOLS" -mountpoint "$TMPMOUNT" -nobrowse + # The "-allowUntrusted" flag has been added to the installer + # command to accomodate for now-expired certificates used + # to sign the downloaded command line tools. + installer -allowUntrusted -pkg "$(find $TMPMOUNT -name '*.mpkg')" -target / + hdiutil detach "$TMPMOUNT" + rm -rf "$TMPMOUNT" + rm "$TOOLS" +fi diff --git a/ansible/roles/package-upgrade/tasks/partials/brew.yml b/ansible/roles/package-upgrade/tasks/partials/brew.yml new file mode 100644 index 000000000..6756493de --- /dev/null +++ b/ansible/roles/package-upgrade/tasks/partials/brew.yml @@ -0,0 +1,51 @@ +--- + +# +# Updates all packages for macos-based distributions +# + + - name: Check for xcode-tools + raw: xcode-select --print-path &> /dev/null + register: xcode + ignore_errors: yes + + - name: Install xcode-tools + script: files/partials/install-xcode.sh + when: xcode.rc > 1 + + - name: Check if Homebrew is already installed + stat: + path: /usr/local/bin/brew + register: brew + + - name: Install Homebrew + become_user: administrator + script: files/partials/install-homebrew.sh + when: not brew.stat.exists + + - name: Upgrade installed packages + become_user: administrator + homebrew: + upgrade_all: yes + + - name: Install brew cu + become_user: administrator + homebrew_tap: + name: buo/cask-upgrade + + - name: Add AdoptOpenJDK Java Repo + become_user: administrator + homebrew_tap: + name: AdoptOpenJDK/openjdk + + - name: Update Casks + become_user: administrator + command: /usr/local/bin/brew cu + + - name: Disable Crash Reporter 1 + become_user: root + command: launchctl unload -w /System/Library/LaunchAgents/com.apple.ReportCrash.plist + + - name: Disable Crash Reporter 2 + become_user: root + command: launchctl unload -w /System/Library/LaunchDaemons/com.apple.ReportCrash.Root.plist diff --git a/ansible/roles/package-upgrade/vars/main.yml b/ansible/roles/package-upgrade/vars/main.yml index 17eaa4fb2..32dbee3ad 100644 --- a/ansible/roles/package-upgrade/vars/main.yml +++ b/ansible/roles/package-upgrade/vars/main.yml @@ -10,6 +10,7 @@ 'dnf': 'fedora', 'pkg': 'freebsd', 'pkgin': 'smartos', + 'brew': 'macos' } # see plugins/filter/filters.py -- second arg is just feedback