From 5409143e1551575526eeabd613a7be773a1be214 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Tue, 4 Apr 2017 21:47:47 -0400 Subject: [PATCH] [FAB-3000] Remove reliance on vagrant baseimage Maintaining the vagrant-baseimage is difficult since we lack suitable automation facilities. It has therefore become an impediment to producing new docker baseimages. In addition, it is also less important these days since most of the heavy lifting is performed by the docker infrastructure. Consider that when the baseimage for vagrant was created, we still needed to compile a bunch of basic tools (golang, protobufs rocksdb, etc). This is no longer true: Some of the tools are now available in binary form, others are exectuted within docker, and others are no longer used (e.g. rocksdb). Therefore, we can greatly simplify our processes by simply eliminating our reliance on a precompiled baseimage for vagrant. Because of the dockerization effort, all we really need is a basic platform with make, git, golang-1.7, and docker. We also include a few other items for developer convenience, such as behave, nodejs, and java. Fixes FAB-3000 Change-Id: Ic0749a1c0361a0b4a83d7ca6879e9d2bb50a5456 Signed-off-by: Gregory Haskins --- devenv/Vagrantfile | 5 +- devenv/setup.sh | 112 ++++++++++++++++++++++++++------------ scripts/install_behave.sh | 34 ++++++++++++ 3 files changed, 111 insertions(+), 40 deletions(-) create mode 100755 scripts/install_behave.sh diff --git a/devenv/Vagrantfile b/devenv/Vagrantfile index fafa8b35d5a..e363555d6e4 100644 --- a/devenv/Vagrantfile +++ b/devenv/Vagrantfile @@ -29,12 +29,9 @@ cd #{SRCMOUNT}/devenv SCRIPT -baseimage_release = File.read '../.baseimage-release' - Vagrant.require_version ">= 1.7.4" Vagrant.configure('2') do |config| - config.vm.box = "hyperledger/fabric-baseimage" - config.vm.box_version = ENV['USE_LOCAL_BASEIMAGE'] ? "0": baseimage_release # Vagrant does not support versioning local images, the local version is always implicitly version 0 + config.vm.box = "ubuntu/xenial64" config.vm.network :forwarded_port, guest: 7050, host: 7050 # fabric orderer service config.vm.network :forwarded_port, guest: 7051, host: 7051 # fabric peer service diff --git a/devenv/setup.sh b/devenv/setup.sh index 0eefee286ec..2d9d406d92b 100755 --- a/devenv/setup.sh +++ b/devenv/setup.sh @@ -1,37 +1,8 @@ #!/bin/bash -helpme() -{ - cat <> /etc/motd +# Update the entire system to the latest releases +apt-get update +#apt-get dist-upgrade -y + +# Install some basic utilities +apt-get install -y build-essential git make curl unzip g++ libtool + +# ---------------------------------------------------------------- +# Install Docker +# ---------------------------------------------------------------- + # Storage backend logic case "${DOCKER_STORAGE_BACKEND}" in aufs|AUFS|"") @@ -59,6 +41,22 @@ case "${DOCKER_STORAGE_BACKEND}" in exit 1;; esac +# Update system +apt-get update -qq + +# Prep apt-get for docker install +apt-get install -y apt-transport-https ca-certificates +apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D + +# Add docker repository +echo deb https://apt.dockerproject.org/repo ubuntu-xenial main > /etc/apt/sources.list.d/docker.list + +# Update system +apt-get update -qq + +# Install docker +apt-get install -y linux-image-extra-$(uname -r) apparmor docker-engine + # Install docker-compose curl -L https://github.com/docker/compose/releases/download/1.8.1/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose @@ -68,19 +66,61 @@ DOCKER_OPTS="-s=${DOCKER_STORAGE_BACKEND_STRING} -r=true --api-cors-header='*' - sed -i.bak '/^DOCKER_OPTS=/{h;s|=.*|=\"'"${DOCKER_OPTS}"'\"|};${x;/^$/{s||DOCKER_OPTS=\"'"${DOCKER_OPTS}"'\"|;H};x}' /etc/default/docker service docker restart -usermod -a -G docker vagrant # Add vagrant user to the docker group +usermod -a -G docker ubuntu # Add ubuntu user to the docker group # Test docker docker run --rm busybox echo All good +# ---------------------------------------------------------------- +# Install Golang +# ---------------------------------------------------------------- +GO_VER=1.7.5 +GO_URL=https://storage.googleapis.com/golang/go${GO_VER}.linux-amd64.tar.gz + # Set Go environment variables needed by other scripts export GOPATH="/opt/gopath" -export GOROOT="/opt/go/" +export GOROOT="/opt/go" PATH=$GOROOT/bin:$GOPATH/bin:$PATH +cat </etc/profile.d/goroot.sh +export GOROOT=$GOROOT +export GOPATH=$GOPATH +export PATH=\$PATH:$GOROOT/bin:$GOPATH/bin +EOF + +mkdir -p $GOROOT + +curl -sL $GO_URL | (cd $GOROOT && tar --strip-components 1 -xz) + +# ---------------------------------------------------------------- +# Install NodeJS +# ---------------------------------------------------------------- +NODE_VER=6.9.5 +NODE_URL=https://nodejs.org/dist/v$NODE_VER/node-v$NODE_VER-linux-x64.tar.gz + +curl -sL $NODE_URL | (cd /usr/local && tar --strip-components 1 -xz ) + +# ---------------------------------------------------------------- +# Install Behave +# ---------------------------------------------------------------- +/hyperledger/scripts/install_behave.sh + +# ---------------------------------------------------------------- +# Install Java +# ---------------------------------------------------------------- +apt-get install -y openjdk-8-jdk maven + +wget https://services.gradle.org/distributions/gradle-2.12-bin.zip -P /tmp --quiet +unzip -q /tmp/gradle-2.12-bin.zip -d /opt && rm /tmp/gradle-2.12-bin.zip +ln -s /opt/gradle-2.12/bin/gradle /usr/bin + +# ---------------------------------------------------------------- +# Misc tasks +# ---------------------------------------------------------------- + # Create directory for the DB sudo mkdir -p /var/hyperledger -sudo chown -R vagrant:vagrant /var/hyperledger +sudo chown -R ubuntu:ubuntu /var/hyperledger # clean any previous builds as they may have image/.dummy files without # the backing docker images (since we are, by definition, rebuilding the @@ -90,7 +130,7 @@ cd $GOPATH/src/github.com/hyperledger/fabric make clean gotools # Ensure permissions are set for GOPATH -sudo chown -R vagrant:vagrant $GOPATH +sudo chown -R ubuntu:ubuntu $GOPATH # Update limits.conf to increase nofiles for RocksDB sudo cp /hyperledger/devenv/limits.conf /etc/security/limits.conf @@ -105,8 +145,8 @@ EOF # Set our shell prompt to something less ugly than the default from packer # Also make it so that it cd's the user to the fabric dir upon logging in -cat <> /home/vagrant/.bashrc -PS1="\u@hyperledger-devenv:v$BASEIMAGE_RELEASE-$DEVENV_REVISION:\w$ " +cat <> /home/ubuntu/.bashrc +PS1="\u@hyperledger-devenv:$DEVENV_REVISION:\w$ " cd $GOPATH/src/github.com/hyperledger/fabric/ EOF diff --git a/scripts/install_behave.sh b/scripts/install_behave.sh new file mode 100755 index 00000000000..8cb082e2c93 --- /dev/null +++ b/scripts/install_behave.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Update system +apt-get update -qq + +# Install Python, pip, behave, nose +# +# install python-dev and libyaml-dev to get compiled speedups +apt-get install --yes python-dev +apt-get install --yes libyaml-dev + +apt-get install --yes python-setuptools +apt-get install --yes python-pip +pip install --upgrade pip +pip install behave +pip install nose + +# updater-server, update-engine, and update-service-common dependencies (for running locally) +pip install -I flask==0.10.1 python-dateutil==2.2 pytz==2014.3 pyyaml==3.10 couchdb==1.0 flask-cors==2.0.1 requests==2.4.3 + +# Python grpc package for behave tests +# Required to update six for grpcio +pip install --ignore-installed six +pip install --upgrade 'grpcio==0.13.1' + +# install ruby and apiaryio +#apt-get install --yes ruby ruby-dev gcc +#gem install apiaryio + +# Install Tcl prerequisites for busywork +apt-get install --yes tcl tclx tcllib + +# Install NPM for the SDK +apt-get install --yes npm