From 2fb1e584d98dbba22592847ab909e0c288760aa5 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Fri, 4 Sep 2015 11:30:33 +0200 Subject: [PATCH] Linux compile on Debian 6. We used to compile on a recent ubuntu version, which is needed for cross-compiling to Windows and OS X. This poses the problem that it links against a relatively recent version of libc, so the resulting binary cannot work on older linux distributions. To work around this issue we tried to compile statically linked binaries. Unfortunately, statically linking against libc breaks in part the name resolution, in particular resolving the names from /etc/hosts. This pull requests makes 2 Docker images: one based on ubuntu for cross compiling to os x and windows and one based on debian 6 for compiling for Linux (both 32 and 64 bits versions). Libpcap continues to be statically linked for Linux. Because the Dockerfiles have drifted quite a bit from the versions from XGO, I moved them to this repository. This closes elastic/packetbeat#225. --- Makefile | 29 ++++- docker/xgo-image-deb6/base/Dockerfile | 44 +++++++ docker/xgo-image-deb6/base/bootstrap.sh | 31 +++++ docker/xgo-image-deb6/base/build.sh | 114 ++++++++++++++++++ docker/xgo-image-deb6/base/build_deps.sh | 27 +++++ docker/xgo-image-deb6/base/fetch.sh | 17 +++ .../xgo-image-deb6/beats-builder/Dockerfile | 18 +++ .../beats-builder/gopacket_pcap.patch | 22 ++++ docker/xgo-image-deb6/build.sh | 5 + docker/xgo-image-deb6/go-1.4.2/Dockerfile | 18 +++ docker/xgo-image/base/Dockerfile | 62 ++++++++++ docker/xgo-image/base/bootstrap.sh | 50 ++++++++ docker/xgo-image/base/build.sh | 114 ++++++++++++++++++ docker/xgo-image/base/build_deps.sh | 27 +++++ docker/xgo-image/base/fetch.sh | 17 +++ .../xgo-image/{ => beats-builder}/Dockerfile | 0 .../{ => beats-builder}/gopacket_pcap.patch | 0 docker/xgo-image/build.sh | 5 + docker/xgo-image/go-1.4.2/Dockerfile | 33 +++++ releases/1.0.0-beta3.yml | 3 + 20 files changed, 630 insertions(+), 6 deletions(-) create mode 100644 docker/xgo-image-deb6/base/Dockerfile create mode 100644 docker/xgo-image-deb6/base/bootstrap.sh create mode 100644 docker/xgo-image-deb6/base/build.sh create mode 100644 docker/xgo-image-deb6/base/build_deps.sh create mode 100644 docker/xgo-image-deb6/base/fetch.sh create mode 100644 docker/xgo-image-deb6/beats-builder/Dockerfile create mode 100644 docker/xgo-image-deb6/beats-builder/gopacket_pcap.patch create mode 100755 docker/xgo-image-deb6/build.sh create mode 100644 docker/xgo-image-deb6/go-1.4.2/Dockerfile create mode 100644 docker/xgo-image/base/Dockerfile create mode 100644 docker/xgo-image/base/bootstrap.sh create mode 100644 docker/xgo-image/base/build.sh create mode 100644 docker/xgo-image/base/build_deps.sh create mode 100644 docker/xgo-image/base/fetch.sh rename docker/xgo-image/{ => beats-builder}/Dockerfile (100%) rename docker/xgo-image/{ => beats-builder}/gopacket_pcap.patch (100%) create mode 100755 docker/xgo-image/build.sh create mode 100644 docker/xgo-image/go-1.4.2/Dockerfile create mode 100644 releases/1.0.0-beta3.yml diff --git a/Makefile b/Makefile index 0126ae7..26d7097 100644 --- a/Makefile +++ b/Makefile @@ -9,16 +9,28 @@ all: packetbeat/deb packetbeat/rpm packetbeat/darwin packetbeat/win packetbeat/b .PHONY: packetbeat topbeat packetbeat topbeat: xgo-image build - cd build && xgo -image=tudorg/beats-builder -static \ + cd build && xgo -image=tudorg/beats-builder \ -before-build=../xgo-scripts/$@_before_build.sh \ -branch $(RELEASE) \ github.com/elastic/$@ -%/deb: % build/god-linux-386 build/god-linux-amd64 fpm-image +.PHONY: packetbeat-linux topbeat-linux +packetbeat-linux: xgo-image build + cd build && xgo -image=tudorg/beats-builder-deb6 \ + -before-build=../xgo-scripts/packetbeat_before_build.sh \ + -branch $(RELEASE) \ + github.com/elastic/packetbeat +topbeat-linux: xgo-image build + cd build && xgo -image=tudorg/beats-builder-deb6 \ + -before-build=../xgo-scripts/topbeat_before_build.sh \ + -branch $(RELEASE) \ + github.com/elastic/topbeat + +%/deb: %-linux build/god-linux-386 build/god-linux-amd64 fpm-image ARCH=386 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/debian/build.sh ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/debian/build.sh -%/rpm: % build/god-linux-386 build/god-linux-amd64 fpm-image +%/rpm: %-linux build/god-linux-386 build/god-linux-amd64 fpm-image ARCH=386 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/centos/build.sh ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/centos/build.sh @@ -28,7 +40,7 @@ packetbeat topbeat: xgo-image build %/win: % fpm-image ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/windows/build.sh -%/bin: % fpm-image +%/bin: %-linux fpm-image ARCH=386 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/binary/build.sh ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/binary/build.sh @@ -39,7 +51,8 @@ deps: .PHONY: xgo-image xgo-image: - docker build -t tudorg/beats-builder docker/xgo-image/ + cd docker/xgo-image/; ./build.sh + cd docker/xgo-image-deb6/; ./build.sh .PHONY: fpm-image fpm-image: @@ -60,11 +73,15 @@ s3-nightlies-upload: all echo $(BUILDID) > build/upload/build_id.txt aws s3 cp --recursive --acl public-read build/upload s3://beats-nightlies +.PHONY: release-upload +release-upload: + aws s3 cp --recursive --acl public-read build/upload s3://download.elasticsearch.org/beats/ + .PHONY: run-interactive run-interactive: docker run -t -i -v $(shell pwd)/build:/build \ -v $(shell pwd)/xgo-scripts/:/scripts \ - --entrypoint=bash tudorg/beats-builder + --entrypoint=bash tudorg/beats-builder-deb6 .PHONY: clean clean: rm -rf build/ || true diff --git a/docker/xgo-image-deb6/base/Dockerfile b/docker/xgo-image-deb6/base/Dockerfile new file mode 100644 index 0000000..3f7e6a5 --- /dev/null +++ b/docker/xgo-image-deb6/base/Dockerfile @@ -0,0 +1,44 @@ +# Go cross compiler (xgo): Base cross-compilation layer +# Copyright (c) 2014 Péter Szilágyi. All rights reserved. +# +# Released under the MIT license. + +FROM debian:6 + +MAINTAINER Péter Szilágyi + +# Configure the Go environment, since it's not going to change +ENV PATH /usr/local/go/bin:$PATH +ENV GOPATH /go + + +# Inject the remote file fetcher and checksum verifier +ADD fetch.sh /fetch.sh +ENV FETCH /fetch.sh +RUN chmod +x $FETCH + + +# Make sure apt-get is up to date and dependent packages are installed +RUN \ + apt-get update && \ + apt-get install -y automake autogen build-essential ca-certificates \ + gcc-multilib \ + clang llvm-dev libtool libxml2-dev uuid-dev libssl-dev pkg-config \ + patch make xz-utils cpio wget unzip git mercurial bzr --no-install-recommends + +# Inject the Go package downloader and tool-chain bootstrapper +ADD bootstrap.sh /bootstrap.sh +ENV BOOTSTRAP /bootstrap.sh +RUN chmod +x $BOOTSTRAP + +# Inject the C dependency cross compiler +ADD build_deps.sh /build_deps.sh +ENV BUILD_DEPS /build_deps.sh +RUN chmod +x $BUILD_DEPS + +# Inject the container entry point, the build script +ADD build.sh /build.sh +ENV BUILD /build.sh +RUN chmod +x $BUILD + +ENTRYPOINT ["/build.sh"] diff --git a/docker/xgo-image-deb6/base/bootstrap.sh b/docker/xgo-image-deb6/base/bootstrap.sh new file mode 100644 index 0000000..ce08177 --- /dev/null +++ b/docker/xgo-image-deb6/base/bootstrap.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Contains the Go tool-chain bootstrapper, that retrieves all the configured +# distribution packages, extracts the binaries and deletes anything not needed. +# +# Usage: bootstrap.sh +# +# Needed environment variables: +# FETCH - Remote file fetcher and checksum verifier (injected by image) +# DIST_LINUX_64, DIST_LINUX_64_SHA1 - 64 bit Linux Go binaries and checksum +# DIST_LINUX_32, DIST_LINUX_32_SHA1 - 32 bit Linux Go binaries and checksum +# DIST_LINUX_ARM, DIST_LINUX_ARM_SHA1 - ARM v5 Linux Go binaries and checksum +# DIST_OSX_64, DIST_OSX_64_SHA1 - 64 bit Mac OSX Go binaries and checksum +# DIST_OSX_32, DIST_OSX_32_SHA1 - 32 bit Mac OSX Go binaries and checksum +# DIST_WIN_64, DIST_WIN_64_SHA1 - 64 bit Windows Go binaries and checksum +# DIST_WIN_32, DIST_WIN_32_SHA1 - 32 bit Windows Go binaries and checksum +set -e + +# Download and verify all the binary packages +$FETCH $DIST_LINUX_64 $DIST_LINUX_64_SHA1 +$FETCH $DIST_LINUX_32 $DIST_LINUX_32_SHA1 + +# Extract the 64 bit Linux package as the primary Go SDK +tar -C /usr/local -xzf `basename $DIST_LINUX_64` + +# Extract all other packages as secondary ones, keeping only the binaries +tar -C /usr/local --wildcards -xzf `basename $DIST_LINUX_32` go/pkg/linux_386* +GOOS=linux GOARCH=386 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap + +# Delete all the intermediate downloaded files +rm -f `basename $DIST_LINUX_64` `basename $DIST_LINUX_32` diff --git a/docker/xgo-image-deb6/base/build.sh b/docker/xgo-image-deb6/base/build.sh new file mode 100644 index 0000000..e849132 --- /dev/null +++ b/docker/xgo-image-deb6/base/build.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# +# Contains the main cross compiler, that individually sets up each target build +# platform, compiles all the C dependencies, then build the requested executable +# itself. +# +# Usage: build.sh +# +# Needed environment variables: +# REPO_REMOTE - Optional VCS remote if not the primary repository is needed +# REPO_BRANCH - Optional VCS branch to use, if not the master branch +# DEPS - Optional list of C dependency packages to build +# PACK - Optional sub-package, if not the import path is being built +# OUT - Optional output prefix to override the package name +# FLAG_V - Optional verbosity flag to set on the Go builder +# FLAG_RACE - Optional race flag to set on the Go builder + +# Download the canonical import path (may fail, don't allow failures beyond) +echo "Fetching main repository $1..." +mkdir -p $GOPATH/src/`dirname $1` +cd $GOPATH/src/`dirname $1` +git clone https://$1.git + +set -e + +cd $GOPATH/src/$1 +export GOPATH=$GOPATH:`pwd`/Godeps/_workspace + +# Switch over the code-base to another checkout if requested +if [ "$REPO_REMOTE" != "" ]; then + echo "Switching over to remote $REPO_REMOTE..." + if [ -d ".git" ]; then + git remote set-url origin $REPO_REMOTE + git pull + elif [ -d ".hg" ]; then + echo -e "[paths]\ndefault = $REPO_REMOTE\n" >> .hg/hgrc + hg pull + fi +fi + +if [ "$REPO_BRANCH" != "" ]; then + echo "Switching over to branch $REPO_BRANCH..." + if [ -d ".git" ]; then + git checkout $REPO_BRANCH + elif [ -d ".hg" ]; then + hg checkout $REPO_BRANCH + fi +fi + +# Download all the C dependencies +echo "Fetching dependencies..." +mkdir -p /deps +DEPS=($DEPS) && for dep in "${DEPS[@]}"; do + echo Downloading $dep + if [ "${dep##*.}" == "tar" ]; then wget -q $dep -O - | tar -C /deps -x; fi + if [ "${dep##*.}" == "gz" ]; then wget -q $dep -O - | tar -C /deps -xz; fi + if [ "${dep##*.}" == "bz2" ]; then wget -q $dep -O - | tar -C /deps -xj; fi +done + +# Configure some global build parameters +NAME=`basename $1/$PACK` +if [ "$OUT" != "" ]; then + NAME=$OUT +fi + + +if [ "$FLAG_V" == "true" ]; then V=-v; fi +if [ "$FLAG_RACE" == "true" ]; then R=-race; fi +if [ "$STATIC" == "true" ]; then LDARGS=--ldflags\ \'-extldflags\ \"-static\"\'; fi + +if [ -n $BEFORE_BUILD ]; then + chmod +x /scripts/$BEFORE_BUILD + echo "Execute /scripts/$BEFORE_BUILD" + /scripts/$BEFORE_BUILD +fi + +# Build for each platform individually +echo "Compiling for linux/amd64..." +HOST=x86_64-linux PREFIX=/usr/local $BUILD_DEPS /deps +GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK +sh -c "GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go build $V $R $LDARGS -o $NAME-linux-amd64$R ./$PACK" + +echo "Compiling for linux/386..." +HOST=i686-linux PREFIX=/usr/local $BUILD_DEPS /deps +GOOS=linux GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK +sh -c "GOOS=linux GOARCH=386 CGO_ENABLED=1 go build $V $LDARGS -o $NAME-linux-386 ./$PACK" + +#echo "Compiling for linux/arm..." +#CC=arm-linux-gnueabi-gcc HOST=arm-linux PREFIX=/usr/local/arm $BUILD_DEPS /deps +#CC=arm-linux-gnueabi-gcc GOOS=linux GOARCH=arm CGO_ENABLED=1 GOARM=5 go get -d ./$PACK +#CC=arm-linux-gnueabi-gcc GOOS=linux GOARCH=arm CGO_ENABLED=1 GOARM=5 go build $V -o $NAME-linux-arm ./$PACK + +#echo "Compiling for windows/amd64..." +#CC=x86_64-w64-mingw32-gcc HOST=x86_64-w64-mingw32 PREFIX=/usr/x86_64-w64-mingw32 $BUILD_DEPS /deps +#CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK +#CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go build $V $R -o $NAME-windows-amd64$R.exe ./$PACK + +#echo "Compiling for windows/386..." +#CC=i686-w64-mingw32-gcc HOST=i686-w64-mingw32 PREFIX=/usr/i686-w64-mingw32 $BUILD_DEPS /deps +#CC=i686-w64-mingw32-gcc GOOS=windows GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK +#CC=i686-w64-mingw32-gcc GOOS=windows GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-windows-386.exe ./$PACK + +#echo "Compiling for darwin/amd64..." +#CC=o64-clang HOST=x86_64-apple-darwin10 PREFIX=/usr/local $BUILD_DEPS /deps +#CC=o64-clang GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK +#CC=o64-clang GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go build $V $R -o $NAME-darwin-amd64$R ./$PACK + +#echo "Compiling for darwin/386..." +#CC=o32-clang HOST=i386-apple-darwin10 PREFIX=/usr/local $BUILD_DEPS /deps +#CC=o32-clang GOOS=darwin GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK +#CC=o32-clang GOOS=darwin GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-darwin-386 ./$PACK + +echo "Moving binaries to host..." +cp `ls -t | head -n 2` /build diff --git a/docker/xgo-image-deb6/base/build_deps.sh b/docker/xgo-image-deb6/base/build_deps.sh new file mode 100644 index 0000000..da00442 --- /dev/null +++ b/docker/xgo-image-deb6/base/build_deps.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Contains the a dependency builder to iterate over all installed dependencies +# and cross compile them to the requested target platform. +# +# Usage: build_deps.sh +# +# Needed environment variables: +# CC - C cross compiler to use for the build +# HOST - Target platform to build (used to find the needed tool-chains) +# PREFIX - File-system path where to install the built binaries +set -e + +# Remove any previous build leftovers, and copy a fresh working set (clean doesn't work for cross compiling) +rm -rf /deps-build && cp -r $1 /deps-build + +# Build all the dependencies (no order for now) +for dep in `ls /deps-build`; do + echo "Configuring dependency $dep for $HOST..." + (cd /deps-build/$dep && ./configure --disable-shared --host=$HOST --prefix=$PREFIX --silent) + + echo "Building dependency $dep for $HOST..." + (cd /deps-build/$dep && make --silent -j install) +done + +# Remove any build artifacts +rm -rf /deps-build diff --git a/docker/xgo-image-deb6/base/fetch.sh b/docker/xgo-image-deb6/base/fetch.sh new file mode 100644 index 0000000..baeae63 --- /dev/null +++ b/docker/xgo-image-deb6/base/fetch.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Contains a simple fetcher to download a file from a remote URL and verify its +# SHA1 checksum. +# +# Usage: fetch.sh +set -e + +# Pull the file from the remote URL +file=`basename $1` +echo "Downloading $1..." +wget --no-check-certificate -q $1 + +# Generate a desired checksum report and check against it +echo "$2 $file" > $file.sum +sha1sum -c $file.sum +rm $file.sum diff --git a/docker/xgo-image-deb6/beats-builder/Dockerfile b/docker/xgo-image-deb6/beats-builder/Dockerfile new file mode 100644 index 0000000..d50fdce --- /dev/null +++ b/docker/xgo-image-deb6/beats-builder/Dockerfile @@ -0,0 +1,18 @@ +FROM tudorg/xgo-deb6-1.4.2 + +MAINTAINER Tudor Golubenco + +# Get libpcap binaries for linux +RUN \ + mkdir -p /libpcap && \ + wget http://ftp.de.debian.org/debian/pool/main/libp/libpcap/libpcap0.8-dev_1.1.1-2+squeeze1_i386.deb && \ + dpkg -x libpcap0.8-dev_*_i386.deb /libpcap/i386 && \ + wget http://ftp.de.debian.org/debian/pool/main/libp/libpcap/libpcap0.8-dev_1.1.1-2+squeeze1_amd64.deb && \ + dpkg -x libpcap0.8-dev_*_amd64.deb /libpcap/amd64 && \ + rm libpcap0.8-dev*.deb +RUN \ + apt-get update && \ + apt-get install -y libpcap0.8-dev + +# add patch for gopacket +ADD gopacket_pcap.patch /gopacket_pcap.patch diff --git a/docker/xgo-image-deb6/beats-builder/gopacket_pcap.patch b/docker/xgo-image-deb6/beats-builder/gopacket_pcap.patch new file mode 100644 index 0000000..379ee18 --- /dev/null +++ b/docker/xgo-image-deb6/beats-builder/gopacket_pcap.patch @@ -0,0 +1,22 @@ +diff --git a/Godeps/_workspace/src/github.com/tsg/gopacket/pcap/pcap.go b/Godeps/_workspace/src/github. +index d2465bb..7b23b84 100644 +--- a/Godeps/_workspace/src/github.com/tsg/gopacket/pcap/pcap.go ++++ b/Godeps/_workspace/src/github.com/tsg/gopacket/pcap/pcap.go +@@ -8,12 +8,13 @@ + package pcap + + /* +-#cgo linux LDFLAGS: -lpcap ++#cgo linux,386 LDFLAGS: /libpcap/i386/usr/lib/libpcap.a ++#cgo linux,amd64 LDFLAGS: /libpcap/amd64/usr/lib/libpcap.a + #cgo freebsd LDFLAGS: -lpcap + #cgo darwin LDFLAGS: -lpcap +-#cgo windows CFLAGS: -I C:/WpdPack/Include +-#cgo windows,386 LDFLAGS: -L C:/WpdPack/Lib -lwpcap +-#cgo windows,amd64 LDFLAGS: -L C:/WpdPack/Lib/x64 -lwpcap ++#cgo windows CFLAGS: -I /libpcap/win/WpdPack/Include ++#cgo windows,386 LDFLAGS: -L /libpcap/win/WpdPack/Lib -lwpcap ++#cgo windows,amd64 LDFLAGS: -L /libpcap/win/WpdPack/Lib/x64 -lwpcap + #include + #include + diff --git a/docker/xgo-image-deb6/build.sh b/docker/xgo-image-deb6/build.sh new file mode 100755 index 0000000..ea46360 --- /dev/null +++ b/docker/xgo-image-deb6/build.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +docker build -t tudorg/xgo-deb6-base base/ && \ + docker build -t tudorg/xgo-deb6-1.4.2 go-1.4.2/ && + docker build -t tudorg/beats-builder-deb6 beats-builder diff --git a/docker/xgo-image-deb6/go-1.4.2/Dockerfile b/docker/xgo-image-deb6/go-1.4.2/Dockerfile new file mode 100644 index 0000000..078b753 --- /dev/null +++ b/docker/xgo-image-deb6/go-1.4.2/Dockerfile @@ -0,0 +1,18 @@ +# Go cross compiler (xgo): Go 1.4.2 layer +# Copyright (c) 2014 Péter Szilágyi. All rights reserved. +# +# Released under the MIT license. + +FROM tudorg/xgo-deb6-base + +MAINTAINER Péter Szilágyi + +# Configure the Go packages and bootstrap them +RUN \ + export DIST_LINUX_64=https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz && \ + export DIST_LINUX_64_SHA1=5020af94b52b65cc9b6f11d50a67e4bae07b0aff && \ + \ + export DIST_LINUX_32=https://storage.googleapis.com/golang/go1.4.2.linux-386.tar.gz && \ + export DIST_LINUX_32_SHA1=50557248e89b6e38d395fda93b2f96b2b860a26a && \ + \ + $BOOTSTRAP diff --git a/docker/xgo-image/base/Dockerfile b/docker/xgo-image/base/Dockerfile new file mode 100644 index 0000000..d9f2145 --- /dev/null +++ b/docker/xgo-image/base/Dockerfile @@ -0,0 +1,62 @@ +# Go cross compiler (xgo): Base cross-compilation layer +# Copyright (c) 2014 Péter Szilágyi. All rights reserved. +# +# Released under the MIT license. + +FROM ubuntu:14.04 + +MAINTAINER Péter Szilágyi + +# Configure the Go environment, since it's not going to change +ENV PATH /usr/local/go/bin:$PATH +ENV GOPATH /go + + +# Inject the remote file fetcher and checksum verifier +ADD fetch.sh /fetch.sh +ENV FETCH /fetch.sh +RUN chmod +x $FETCH + + +# Make sure apt-get is up to date and dependent packages are installed +RUN \ + apt-get update && \ + apt-get install -y automake autogen build-essential ca-certificates \ + gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-multilib gcc-mingw-w64 \ + clang llvm-dev libtool libxml2-dev uuid-dev libssl-dev pkg-config \ + patch make xz-utils cpio wget unzip git mercurial bzr --no-install-recommends + +# Configure the container for OSX cross compilation +ENV OSX_SDK_PATH https://github.com/trevd/android_platform_build2/raw/master/osxsdks10.6.tar.gz +ENV OSX_SDK MacOSX10.6.sdk + +RUN \ + git clone https://github.com/tpoechtrager/osxcross.git && \ + \ + $FETCH $OSX_SDK_PATH f526b4ae9806e8d31e3b094e3f004f8f160a3fad + +RUN \ + tar -xzf `basename $OSX_SDK_PATH` --strip-components 1 SDKs/$OSX_SDK && \ + tar -cjf /osxcross/tarballs/$OSX_SDK.tar.bz2 $OSX_SDK && \ + rm -rf `basename $OSX_SDK_PATH` $OSX_SDK && \ + \ + UNATTENDED=yes /osxcross/build.sh +ENV PATH /osxcross/target/bin:$PATH + + +# Inject the Go package downloader and tool-chain bootstrapper +ADD bootstrap.sh /bootstrap.sh +ENV BOOTSTRAP /bootstrap.sh +RUN chmod +x $BOOTSTRAP + +# Inject the C dependency cross compiler +ADD build_deps.sh /build_deps.sh +ENV BUILD_DEPS /build_deps.sh +RUN chmod +x $BUILD_DEPS + +# Inject the container entry point, the build script +ADD build.sh /build.sh +ENV BUILD /build.sh +RUN chmod +x $BUILD + +ENTRYPOINT ["/build.sh"] diff --git a/docker/xgo-image/base/bootstrap.sh b/docker/xgo-image/base/bootstrap.sh new file mode 100644 index 0000000..24c11ab --- /dev/null +++ b/docker/xgo-image/base/bootstrap.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Contains the Go tool-chain bootstrapper, that retrieves all the configured +# distribution packages, extracts the binaries and deletes anything not needed. +# +# Usage: bootstrap.sh +# +# Needed environment variables: +# FETCH - Remote file fetcher and checksum verifier (injected by image) +# DIST_LINUX_64, DIST_LINUX_64_SHA1 - 64 bit Linux Go binaries and checksum +# DIST_LINUX_32, DIST_LINUX_32_SHA1 - 32 bit Linux Go binaries and checksum +# DIST_LINUX_ARM, DIST_LINUX_ARM_SHA1 - ARM v5 Linux Go binaries and checksum +# DIST_OSX_64, DIST_OSX_64_SHA1 - 64 bit Mac OSX Go binaries and checksum +# DIST_OSX_32, DIST_OSX_32_SHA1 - 32 bit Mac OSX Go binaries and checksum +# DIST_WIN_64, DIST_WIN_64_SHA1 - 64 bit Windows Go binaries and checksum +# DIST_WIN_32, DIST_WIN_32_SHA1 - 32 bit Windows Go binaries and checksum +set -e + +# Download and verify all the binary packages +$FETCH $DIST_LINUX_64 $DIST_LINUX_64_SHA1 +$FETCH $DIST_LINUX_32 $DIST_LINUX_32_SHA1 +$FETCH $DIST_LINUX_ARM $DIST_LINUX_ARM_SHA1 +$FETCH $DIST_OSX_64 $DIST_OSX_64_SHA1 +$FETCH $DIST_OSX_32 $DIST_OSX_32_SHA1 +$FETCH $DIST_WIN_64 $DIST_WIN_64_SHA1 +$FETCH $DIST_WIN_32 $DIST_WIN_32_SHA1 + +# Extract the 64 bit Linux package as the primary Go SDK +tar -C /usr/local -xzf `basename $DIST_LINUX_64` + +# Extract all other packages as secondary ones, keeping only the binaries +tar -C /usr/local --wildcards -xzf `basename $DIST_LINUX_32` go/pkg/linux_386* +GOOS=linux GOARCH=386 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap +tar -C /usr/local --wildcards -xzf `basename $DIST_LINUX_ARM` go/pkg/linux_arm* +GOOS=linux GOARCH=arm /usr/local/go/pkg/tool/linux_amd64/dist bootstrap + +tar -C /usr/local --wildcards -xzf `basename $DIST_OSX_64` go/pkg/darwin_amd64* +GOOS=darwin GOARCH=amd64 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap +tar -C /usr/local --wildcards -xzf `basename $DIST_OSX_32` go/pkg/darwin_386* +GOOS=darwin GOARCH=386 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap + +unzip -d /usr/local -q `basename $DIST_WIN_64` go/pkg/windows_amd64* +GOOS=windows GOARCH=amd64 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap +unzip -d /usr/local -q `basename $DIST_WIN_32` go/pkg/windows_386* +GOOS=windows GOARCH=386 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap + +# Delete all the intermediate downloaded files +rm -f `basename $DIST_LINUX_64` `basename $DIST_LINUX_32` `basename $DIST_LINUX_ARM` \ + `basename $DIST_OSX_64` `basename $DIST_OSX_32` \ + `basename $DIST_WIN_64` `basename $DIST_WIN_32` diff --git a/docker/xgo-image/base/build.sh b/docker/xgo-image/base/build.sh new file mode 100644 index 0000000..c6f7f03 --- /dev/null +++ b/docker/xgo-image/base/build.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# +# Contains the main cross compiler, that individually sets up each target build +# platform, compiles all the C dependencies, then build the requested executable +# itself. +# +# Usage: build.sh +# +# Needed environment variables: +# REPO_REMOTE - Optional VCS remote if not the primary repository is needed +# REPO_BRANCH - Optional VCS branch to use, if not the master branch +# DEPS - Optional list of C dependency packages to build +# PACK - Optional sub-package, if not the import path is being built +# OUT - Optional output prefix to override the package name +# FLAG_V - Optional verbosity flag to set on the Go builder +# FLAG_RACE - Optional race flag to set on the Go builder + +# Download the canonical import path (may fail, don't allow failures beyond) +echo "Fetching main repository $1..." +mkdir -p $GOPATH/src/`dirname $1` +cd $GOPATH/src/`dirname $1` +git clone https://$1.git + +set -e + +cd $GOPATH/src/$1 +export GOPATH=$GOPATH:`pwd`/Godeps/_workspace + +# Switch over the code-base to another checkout if requested +if [ "$REPO_REMOTE" != "" ]; then + echo "Switching over to remote $REPO_REMOTE..." + if [ -d ".git" ]; then + git remote set-url origin $REPO_REMOTE + git pull + elif [ -d ".hg" ]; then + echo -e "[paths]\ndefault = $REPO_REMOTE\n" >> .hg/hgrc + hg pull + fi +fi + +if [ "$REPO_BRANCH" != "" ]; then + echo "Switching over to branch $REPO_BRANCH..." + if [ -d ".git" ]; then + git checkout $REPO_BRANCH + elif [ -d ".hg" ]; then + hg checkout $REPO_BRANCH + fi +fi + +# Download all the C dependencies +echo "Fetching dependencies..." +mkdir -p /deps +DEPS=($DEPS) && for dep in "${DEPS[@]}"; do + echo Downloading $dep + if [ "${dep##*.}" == "tar" ]; then wget -q $dep -O - | tar -C /deps -x; fi + if [ "${dep##*.}" == "gz" ]; then wget -q $dep -O - | tar -C /deps -xz; fi + if [ "${dep##*.}" == "bz2" ]; then wget -q $dep -O - | tar -C /deps -xj; fi +done + +# Configure some global build parameters +NAME=`basename $1/$PACK` +if [ "$OUT" != "" ]; then + NAME=$OUT +fi + + +if [ "$FLAG_V" == "true" ]; then V=-v; fi +if [ "$FLAG_RACE" == "true" ]; then R=-race; fi +if [ "$STATIC" == "true" ]; then LDARGS=--ldflags\ \'-extldflags\ \"-static\"\'; fi + +if [ -n $BEFORE_BUILD ]; then + chmod +x /scripts/$BEFORE_BUILD + echo "Execute /scripts/$BEFORE_BUILD" + /scripts/$BEFORE_BUILD +fi + +# Build for each platform individually +#echo "Compiling for linux/amd64..." +#HOST=x86_64-linux PREFIX=/usr/local $BUILD_DEPS /deps +#GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK +#sh -c "GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go build $V $R $LDARGS -o $NAME-linux-amd64$R ./$PACK" + +#echo "Compiling for linux/386..." +#HOST=i686-linux PREFIX=/usr/local $BUILD_DEPS /deps +#GOOS=linux GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK +#sh -c "GOOS=linux GOARCH=386 CGO_ENABLED=1 go build $V $LDARGS -o $NAME-linux-386 ./$PACK" + +#echo "Compiling for linux/arm..." +#CC=arm-linux-gnueabi-gcc HOST=arm-linux PREFIX=/usr/local/arm $BUILD_DEPS /deps +#CC=arm-linux-gnueabi-gcc GOOS=linux GOARCH=arm CGO_ENABLED=1 GOARM=5 go get -d ./$PACK +#CC=arm-linux-gnueabi-gcc GOOS=linux GOARCH=arm CGO_ENABLED=1 GOARM=5 go build $V -o $NAME-linux-arm ./$PACK + +echo "Compiling for windows/amd64..." +CC=x86_64-w64-mingw32-gcc HOST=x86_64-w64-mingw32 PREFIX=/usr/x86_64-w64-mingw32 $BUILD_DEPS /deps +CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK +CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go build $V $R -o $NAME-windows-amd64$R.exe ./$PACK + +echo "Compiling for windows/386..." +CC=i686-w64-mingw32-gcc HOST=i686-w64-mingw32 PREFIX=/usr/i686-w64-mingw32 $BUILD_DEPS /deps +CC=i686-w64-mingw32-gcc GOOS=windows GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK +CC=i686-w64-mingw32-gcc GOOS=windows GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-windows-386.exe ./$PACK + +echo "Compiling for darwin/amd64..." +CC=o64-clang HOST=x86_64-apple-darwin10 PREFIX=/usr/local $BUILD_DEPS /deps +CC=o64-clang GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK +CC=o64-clang GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go build $V $R -o $NAME-darwin-amd64$R ./$PACK + +#echo "Compiling for darwin/386..." +#CC=o32-clang HOST=i386-apple-darwin10 PREFIX=/usr/local $BUILD_DEPS /deps +#CC=o32-clang GOOS=darwin GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK +#CC=o32-clang GOOS=darwin GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-darwin-386 ./$PACK + +echo "Moving binaries to host..." +cp `ls -t | head -n 3` /build diff --git a/docker/xgo-image/base/build_deps.sh b/docker/xgo-image/base/build_deps.sh new file mode 100644 index 0000000..da00442 --- /dev/null +++ b/docker/xgo-image/base/build_deps.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Contains the a dependency builder to iterate over all installed dependencies +# and cross compile them to the requested target platform. +# +# Usage: build_deps.sh +# +# Needed environment variables: +# CC - C cross compiler to use for the build +# HOST - Target platform to build (used to find the needed tool-chains) +# PREFIX - File-system path where to install the built binaries +set -e + +# Remove any previous build leftovers, and copy a fresh working set (clean doesn't work for cross compiling) +rm -rf /deps-build && cp -r $1 /deps-build + +# Build all the dependencies (no order for now) +for dep in `ls /deps-build`; do + echo "Configuring dependency $dep for $HOST..." + (cd /deps-build/$dep && ./configure --disable-shared --host=$HOST --prefix=$PREFIX --silent) + + echo "Building dependency $dep for $HOST..." + (cd /deps-build/$dep && make --silent -j install) +done + +# Remove any build artifacts +rm -rf /deps-build diff --git a/docker/xgo-image/base/fetch.sh b/docker/xgo-image/base/fetch.sh new file mode 100644 index 0000000..d0b5039 --- /dev/null +++ b/docker/xgo-image/base/fetch.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Contains a simple fetcher to download a file from a remote URL and verify its +# SHA1 checksum. +# +# Usage: fetch.sh +set -e + +# Pull the file from the remote URL +file=`basename $1` +echo "Downloading $1..." +wget -q $1 + +# Generate a desired checksum report and check against it +echo "$2 $file" > $file.sum +sha1sum -c $file.sum +rm $file.sum diff --git a/docker/xgo-image/Dockerfile b/docker/xgo-image/beats-builder/Dockerfile similarity index 100% rename from docker/xgo-image/Dockerfile rename to docker/xgo-image/beats-builder/Dockerfile diff --git a/docker/xgo-image/gopacket_pcap.patch b/docker/xgo-image/beats-builder/gopacket_pcap.patch similarity index 100% rename from docker/xgo-image/gopacket_pcap.patch rename to docker/xgo-image/beats-builder/gopacket_pcap.patch diff --git a/docker/xgo-image/build.sh b/docker/xgo-image/build.sh new file mode 100755 index 0000000..997f474 --- /dev/null +++ b/docker/xgo-image/build.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +docker build -t tudorg/xgo-base base/ && \ + docker build -t tudorg/xgo-1.4.2 go-1.4.2/ && + docker build -t tudorg/beats-builder beats-builder diff --git a/docker/xgo-image/go-1.4.2/Dockerfile b/docker/xgo-image/go-1.4.2/Dockerfile new file mode 100644 index 0000000..6a02602 --- /dev/null +++ b/docker/xgo-image/go-1.4.2/Dockerfile @@ -0,0 +1,33 @@ +# Go cross compiler (xgo): Go 1.4.2 layer +# Copyright (c) 2014 Péter Szilágyi. All rights reserved. +# +# Released under the MIT license. + +FROM tudorg/xgo-base + +MAINTAINER Péter Szilágyi + +# Configure the Go packages and bootstrap them +RUN \ + export DIST_LINUX_64=https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz && \ + export DIST_LINUX_64_SHA1=5020af94b52b65cc9b6f11d50a67e4bae07b0aff && \ + \ + export DIST_LINUX_32=https://storage.googleapis.com/golang/go1.4.2.linux-386.tar.gz && \ + export DIST_LINUX_32_SHA1=50557248e89b6e38d395fda93b2f96b2b860a26a && \ + \ + export DIST_LINUX_ARM=http://dave.cheney.net/paste/go1.4.2.linux-arm~armv5-1.tar.gz && \ + export DIST_LINUX_ARM_SHA1=1bcfc8ef9c2aa381722b71b8c8d83cb58e973116 && \ + \ + export DIST_OSX_64=https://storage.googleapis.com/golang/go1.4.2.darwin-amd64-osx10.6.tar.gz && \ + export DIST_OSX_64_SHA1=00c3f9a03daff818b2132ac31d57f054925c60e7 && \ + \ + export DIST_OSX_32=https://storage.googleapis.com/golang/go1.4.2.darwin-386-osx10.6.tar.gz && \ + export DIST_OSX_32_SHA1=fb3e6b30f4e1b1be47bbb98d79dd53da8dec24ec && \ + \ + export DIST_WIN_64=https://storage.googleapis.com/golang/go1.4.2.windows-amd64.zip && \ + export DIST_WIN_64_SHA1=91b229a3ff0a1ce6e791c832b0b4670bfc5457b5 && \ + \ + export DIST_WIN_32=https://storage.googleapis.com/golang/go1.4.2.windows-386.zip && \ + export DIST_WIN_32_SHA1=0e074e66a7816561d7947ff5c3514be96f347dc4 && \ + \ + $BOOTSTRAP diff --git a/releases/1.0.0-beta3.yml b/releases/1.0.0-beta3.yml new file mode 100644 index 0000000..9059eee --- /dev/null +++ b/releases/1.0.0-beta3.yml @@ -0,0 +1,3 @@ +version: 1.0.0-beta3 +rpm_version: 1.0.0~beta3 +deb_version: 1.0.0~beta3