From 497f31b24b8ec4075c597d011ef8640ae80bade9 Mon Sep 17 00:00:00 2001 From: "J. Emrys Landivar" Date: Wed, 26 May 2021 12:18:19 -0500 Subject: [PATCH] chore: nightly builds, now with rust for flux. (#2558) --- Dockerfile_build_ubuntu64 | 25 +++++----------- build.py | 62 +++++++++++++++++++-------------------- build.sh | 13 ++++---- go.mod | 1 + test.sh | 12 +++++--- tools.go | 1 + 6 files changed, 56 insertions(+), 58 deletions(-) diff --git a/Dockerfile_build_ubuntu64 b/Dockerfile_build_ubuntu64 index fb037d0df..9f7e7aef4 100644 --- a/Dockerfile_build_ubuntu64 +++ b/Dockerfile_build_ubuntu64 @@ -1,4 +1,5 @@ -FROM ubuntu:18.04 +ARG GO_VERSION +FROM quay.io/influxdb/cross-builder:go${GO_VERSION}-latest # This dockerfile is capabable of performing all # build/test/package/deploy actions needed for Kapacitor. @@ -41,35 +42,23 @@ RUN wget -q https://github.com/google/protobuf/releases/download/v${PROTO_VERSIO # Install protobuf3 python library RUN wget -q https://github.com/google/protobuf/releases/download/v${PROTO_VERSION}/protobuf-python-${PROTO_VERSION}.tar.gz \ && tar -xf protobuf-python-${PROTO_VERSION}.tar.gz \ - && cd /protobuf-${PROTO_VERSION}/python \ + && cd protobuf-${PROTO_VERSION}/python \ && python2 setup.py install \ && python3 setup.py install \ + && cd ../../ \ && rm -rf /protobuf-${PROTO_VERSION} protobuf-python-${PROTO_VERSION}.tar.gz -# Install rust -ENV PATH "/root/.cargo/bin:$PATH" -RUN curl --proto '=https' --tlsv1.2 --tls-max 1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain stable -y && \ - cargo --help - -# Install go -ENV GOPATH /root/go -ENV GO_VERSION 1.15.5 -ENV GO_ARCH amd64 -RUN wget -q https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz; \ - tar -C /usr/local/ -xf /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz ; \ - rm /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz -ENV PATH /usr/local/go/bin:$PATH ENV PROJECT_DIR $GOPATH/src/github.com/influxdata/kapacitor -ENV PKG_CONFIG $PROJECT_DIR/pkg-config.sh +#ENV PKG_CONFIG $PROJECT_DIR/pkg-config.sh ENV PATH $GOPATH/bin:$PATH RUN mkdir -p $PROJECT_DIR WORKDIR $PROJECT_DIR VOLUME $PROJECT_DIR -VOLUME /root/go/src +VOLUME /go/src # Configure local git RUN git config --global user.email "support@influxdb.com" RUN git config --global user.Name "Docker Builder" -ENTRYPOINT [ "/root/go/src/github.com/influxdata/kapacitor/build.py" ] +ENTRYPOINT [ "/go/src/github.com/influxdata/kapacitor/build.py" ] diff --git a/build.py b/build.py index 600e3d8b9..2a5d64027 100755 --- a/build.py +++ b/build.py @@ -93,8 +93,8 @@ supported_builds = { 'darwin': [ "amd64" ], - 'linux': [ "amd64", "i386", "armhf", "arm64", "armel", "static_i386", "static_amd64" ], - 'windows': [ "amd64", "i386" ] + 'linux': [ "arm64", "amd64" ], + 'windows': [ "amd64" ] } supported_packages = { @@ -158,7 +158,11 @@ def run_generate(): """Run 'go generate' to rebuild any static assets. """ logging.info("Running generate...") - run("go install -mod=mod github.com/golang/protobuf/protoc-gen-go github.com/benbjohnson/tmpl github.com/mailru/easyjson/easyjson") + run("""go install -mod=mod + github.com/golang/protobuf/protoc-gen-go \ + github.com/benbjohnson/tmpl \ + github.com/mailru/easyjson/easyjson \ + github.com/influxdata/pkg-config""") try: subprocess.check_output(["go", "generate", "./..."]) except subprocess.CalledProcessError as exc: @@ -382,13 +386,11 @@ def get_system_arch(): arch = os.uname()[4] if arch == "x86_64": arch = "amd64" - elif arch == "386": - arch = "i386" elif arch == "aarch64": arch = "arm64" elif 'arm' in arch: # Prevent uname from reporting full ARM arch (eg 'armv7l') - arch = "arm" + arch = "arm64" return arch def get_system_platform(): @@ -491,6 +493,7 @@ def build(version=None, nightly=False, race=False, clean=False, + cc="", outdir=".", tags=[], static=False): @@ -522,42 +525,39 @@ def build(version=None, logging.info("Building target: {}".format(target)) build_command = "" - # Handle static binary output - if static is True or "static_" in arch: - if "static_" in arch: - static = True - arch = arch.replace("static_", "") - build_command += "CGO_ENABLED=0 " + build_command += "CGO_ENABLED=1 " # Handle variations in architecture output fullarch = arch - if arch == "i386" or arch == "i686": - arch = "386" - elif arch == "aarch64" or arch == "arm64": + if arch == "aarch64" or arch == "arm64": arch = "arm64" - elif "arm" in arch: - arch = "arm" - build_command += "GOOS={} GOARCH={} ".format(platform, arch) + + if platform == "linux": + if arch == "amd64": + tags += ["netgo", "osusergo", "static_build"] + if arch == "arm64": + cc = "aarch64-unknown-linux-musl-cc" + tags += ["netgo", "osusergo", "static_build", "noasm"] + elif platform == "darwin" and arch == "amd64": + cc = "x86_64-apple-darwin15-clang" + tags += [ "netgo", "osusergo"] + elif platform == "windows" and arch == "amd64": + cc = "x86_64-w64-mingw32-gcc" + build_command += "CC={} GOOS={} GOARCH={} ".format(cc, platform, arch) if "arm" in fullarch: - if fullarch == "armel": - build_command += "GOARM=5 " - elif fullarch == "armhf" or fullarch == "arm": - build_command += "GOARM=6 " - elif fullarch == "arm64": - # GOARM not used - see https://github.com/golang/go/wiki/GoArm - pass - else: - logging.error("Invalid ARM architecture specified: {}".format(arch)) - logging.error("Please specify either 'armel', 'armhf', or 'arm64'.") + if fullarch != "arm64": + logging.error("Invalid ARM architecture specified: {} only arm64 is supported".format(arch)) return False if platform == 'windows': target = target + '.exe' - build_command += "go build -o {} ".format(os.path.join(outdir, target)) + build_command += "go build -buildmode=exe -o {} ".format(os.path.join(outdir, target)) + else: + build_command += "go build -o {} ".format(os.path.join(outdir, target)) if race: build_command += "-race " if len(tags) > 0: - build_command += "-tags {} ".format(','.join(tags)) + build_command += "-tags \"{}\" ".format(' '.join(tags)) if "1.4" in get_go_version(): if static: build_command += "-ldflags=\"-s -X main.version {} -X main.branch {} -X main.commit {} -X main.platform OSS\" ".format(version, @@ -913,7 +913,7 @@ def main(args): type=str, help='Name to use for package name (when package is specified)') parser.add_argument('--arch', - metavar='', + metavar='', type=str, default=get_system_arch(), help='Target architecture for build output') diff --git a/build.sh b/build.sh index b349c4026..467c58ca5 100755 --- a/build.sh +++ b/build.sh @@ -12,21 +12,24 @@ BUILD_NUM=${BUILD_NUM-$RANDOM} # Home dir of the docker user HOME_DIR=/root +GO_VERSION=1.15.10 + imagename=kapacitor-builder-img-$BUILD_NUM dataname=kapacitor-data-$BUILD_NUM # Build new docker image -docker build -f Dockerfile_build_ubuntu64 -t $imagename $DIR +docker build -f Dockerfile_build_ubuntu64 -t $imagename --build-arg GO_VERSION=${GO_VERSION} $DIR # Build new docker image -docker build -f Dockerfile_build_ubuntu64 -t influxdata/kapacitor-builder $DIR +docker build -f Dockerfile_build_ubuntu64 -t influxdata/kapacitor-builder --build-arg GO_VERSION=${GO_VERSION} $DIR # Create data volume with code docker create \ --name $dataname \ - -v "$HOME_DIR/go/src/github.com/influxdata/kapacitor" \ + -v "/go/src/github.com/influxdata/kapacitor" \ $imagename /bin/true -docker cp "$DIR/" "$dataname:$HOME_DIR/go/src/github.com/influxdata/" + +docker cp "$DIR/" "$dataname:/go/src/github.com/influxdata/" echo "Running build.py" # Run docker @@ -38,6 +41,6 @@ docker run \ $imagename \ "$@" -docker cp "$dataname:$HOME_DIR/go/src/github.com/influxdata/kapacitor/build" \ +docker cp "$dataname:/go/src/github.com/influxdata/kapacitor/build" \ ./ docker rm -v $dataname diff --git a/go.mod b/go.mod index 635f88494..e530f6e26 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( github.com/influxdata/influx-cli/v2 v2.0.0-20210526124422-63da8eccbdb7 github.com/influxdata/influxdb v1.8.4 github.com/influxdata/influxdb/v2 v2.0.1-alpha.10.0.20210507184756-dc72dc3f0c07 + github.com/influxdata/pkg-config v0.2.7 github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 github.com/influxdata/wlog v0.0.0-20160411224016-7c63b0a71ef8 github.com/k-sone/snmpgo v3.2.0+incompatible diff --git a/test.sh b/test.sh index f509691d4..36a6e4d30 100755 --- a/test.sh +++ b/test.sh @@ -34,6 +34,10 @@ TIMEOUT=${TIMEOUT-1000s} NO_UNCOMMITTED=${NO_UNCOMMITTED-false} # Home dir of the docker user HOME_DIR=/root +# Go version +GO_VERSION=1.15.10 +# GOPATH +GOPATH=/go no_uncomitted_arg="$no_uncommitted_arg" if [ ! $NO_UNCOMMITTED ] @@ -66,16 +70,16 @@ function run_test_docker { dataname="kapacitor-data-$BUILD_NUM" echo "Building docker image $imagename" - docker build -f "$dockerfile" -t "$imagename" . + docker build -f "$dockerfile" --build-arg GO_VERSION=$GO_VERSION -t "$imagename" . echo "Running test in docker $name with args $@" # Create data volume with code docker create \ --name $dataname \ - -v "$HOME_DIR/go/src/github.com/influxdata/kapacitor" \ + -v "$GOPATH/src/github.com/influxdata/kapacitor" \ $imagename /bin/true - docker cp "$DIR/" "$dataname:$HOME_DIR/go/src/github.com/influxdata/" + docker cp "$DIR/" "$dataname:$GOPATH/src/github.com/influxdata/" # Run tests in docker docker run \ @@ -92,7 +96,7 @@ function run_test_docker { # Copy results back out docker cp \ - "$dataname:$HOME_DIR/go/src/github.com/influxdata/kapacitor/build" \ + "$dataname:$GOPATH/src/github.com/influxdata/kapacitor/build" \ ./ # Remove the data and builder containers diff --git a/tools.go b/tools.go index 04bccaa53..cff26dff0 100644 --- a/tools.go +++ b/tools.go @@ -8,5 +8,6 @@ package kapacitor import ( _ "github.com/benbjohnson/tmpl" _ "github.com/golang/protobuf/protoc-gen-go" + _ "github.com/influxdata/pkg-config" _ "github.com/mailru/easyjson/easyjson" )