chore: nightly builds, now with rust for flux. (#2558)

Dockerfile_build_ubuntu64

@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG GO_VERSION
+FROM quay.io/influxdb/cross-builder:go${GO_VERSION}-latest
 
 # This dockerfile is capabable of performing all
 # build/test/package/deploy actions needed for Kapacitor.
@@ -41,35 +42,23 @@ RUN wget -q https://github.com/google/protobuf/releases/download/v${PROTO_VERSIO
 # Install protobuf3 python library
 RUN wget -q https://github.com/google/protobuf/releases/download/v${PROTO_VERSION}/protobuf-python-${PROTO_VERSION}.tar.gz \
     && tar -xf protobuf-python-${PROTO_VERSION}.tar.gz \
-    && cd /protobuf-${PROTO_VERSION}/python \
+    && cd protobuf-${PROTO_VERSION}/python \
     && python2 setup.py install \
     && python3 setup.py install \
+    && cd ../../ \
     && rm -rf /protobuf-${PROTO_VERSION} protobuf-python-${PROTO_VERSION}.tar.gz
 
-# Install rust
-ENV PATH "/root/.cargo/bin:$PATH"
-RUN curl --proto '=https' --tlsv1.2 --tls-max 1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain stable -y && \
-    cargo --help
-
-# Install go
-ENV GOPATH /root/go
-ENV GO_VERSION 1.15.5
-ENV GO_ARCH amd64
-RUN wget -q https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz; \
-   tar -C /usr/local/ -xf /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz ; \
-   rm /go${GO_VERSION}.linux-${GO_ARCH}.tar.gz
-ENV PATH /usr/local/go/bin:$PATH
 ENV PROJECT_DIR $GOPATH/src/github.com/influxdata/kapacitor
-ENV PKG_CONFIG  $PROJECT_DIR/pkg-config.sh
+#ENV PKG_CONFIG  $PROJECT_DIR/pkg-config.sh
 ENV PATH $GOPATH/bin:$PATH
 RUN mkdir -p $PROJECT_DIR
 WORKDIR $PROJECT_DIR
 
 VOLUME $PROJECT_DIR
-VOLUME /root/go/src
+VOLUME /go/src
 
 # Configure local git
 RUN git config --global user.email "support@influxdb.com"
 RUN git config --global user.Name "Docker Builder"
 
-ENTRYPOINT [ "/root/go/src/github.com/influxdata/kapacitor/build.py" ]
+ENTRYPOINT [ "/go/src/github.com/influxdata/kapacitor/build.py" ]

build.py

@@ -93,8 +93,8 @@
 
 supported_builds = {
     'darwin': [ "amd64" ],
-    'linux': [ "amd64", "i386", "armhf", "arm64", "armel", "static_i386", "static_amd64" ],
-    'windows': [ "amd64", "i386" ]
+    'linux': [ "arm64", "amd64" ],
+    'windows': [ "amd64" ]
 }
 
 supported_packages = {
@@ -158,7 +158,11 @@ def run_generate():
     """Run 'go generate' to rebuild any static assets.
     """
     logging.info("Running generate...")
-    run("go install -mod=mod github.com/golang/protobuf/protoc-gen-go github.com/benbjohnson/tmpl github.com/mailru/easyjson/easyjson")
+    run("""go install -mod=mod 
+        github.com/golang/protobuf/protoc-gen-go \
+        github.com/benbjohnson/tmpl \
+        github.com/mailru/easyjson/easyjson \
+        github.com/influxdata/pkg-config""")
     try:
          subprocess.check_output(["go", "generate", "./..."])
     except subprocess.CalledProcessError as exc:
@@ -382,13 +386,11 @@ def get_system_arch():
     arch = os.uname()[4]
     if arch == "x86_64":
         arch = "amd64"
-    elif arch == "386":
-        arch = "i386"
     elif arch == "aarch64":
         arch = "arm64"
     elif 'arm' in arch:
         # Prevent uname from reporting full ARM arch (eg 'armv7l')
-        arch = "arm"
+        arch = "arm64"
     return arch
 
 def get_system_platform():
@@ -491,6 +493,7 @@ def build(version=None,
           nightly=False,
           race=False,
           clean=False,
+          cc="",
           outdir=".",
           tags=[],
           static=False):
@@ -522,42 +525,39 @@ def build(version=None,
         logging.info("Building target: {}".format(target))
         build_command = ""
 
-        # Handle static binary output
-        if static is True or "static_" in arch:
-            if "static_" in arch:
-                static = True
-                arch = arch.replace("static_", "")
-            build_command += "CGO_ENABLED=0 "
+        build_command += "CGO_ENABLED=1 "
 
         # Handle variations in architecture output
         fullarch = arch
-        if arch == "i386" or arch == "i686":
-            arch = "386"
-        elif arch == "aarch64" or arch == "arm64":
+        if  arch == "aarch64" or arch == "arm64":
             arch = "arm64"
-        elif "arm" in arch:
-            arch = "arm"
-        build_command += "GOOS={} GOARCH={} ".format(platform, arch)
+
+        if platform == "linux":
+            if arch == "amd64":
+                tags += ["netgo", "osusergo", "static_build"]
+            if arch == "arm64":
+                cc = "aarch64-unknown-linux-musl-cc"
+                tags += ["netgo", "osusergo", "static_build", "noasm"]
+        elif platform == "darwin" and arch == "amd64":
+            cc = "x86_64-apple-darwin15-clang"
+            tags += [ "netgo", "osusergo"]
+        elif  platform == "windows" and arch == "amd64":
+            cc = "x86_64-w64-mingw32-gcc"
+        build_command += "CC={} GOOS={} GOARCH={} ".format(cc, platform, arch)
 
         if "arm" in fullarch:
-            if fullarch == "armel":
-                build_command += "GOARM=5 "
-            elif fullarch == "armhf" or fullarch == "arm":
-                build_command += "GOARM=6 "
-            elif fullarch == "arm64":
-                # GOARM not used - see https://github.com/golang/go/wiki/GoArm
-                pass
-            else:
-                logging.error("Invalid ARM architecture specified: {}".format(arch))
-                logging.error("Please specify either 'armel', 'armhf', or 'arm64'.")
+            if  fullarch != "arm64":
+                logging.error("Invalid ARM architecture specified: {} only arm64 is supported".format(arch))
                 return False
         if platform == 'windows':
             target = target + '.exe'
-        build_command += "go build -o {} ".format(os.path.join(outdir, target))
+            build_command += "go build -buildmode=exe -o {} ".format(os.path.join(outdir, target))
+        else:
+            build_command += "go build -o {} ".format(os.path.join(outdir, target))
         if race:
             build_command += "-race "
         if len(tags) > 0:
-            build_command += "-tags {} ".format(','.join(tags))
+            build_command += "-tags \"{}\" ".format(' '.join(tags))
         if "1.4" in get_go_version():
             if static:
                 build_command += "-ldflags=\"-s -X main.version {} -X main.branch {} -X main.commit {} -X main.platform OSS\" ".format(version,
@@ -913,7 +913,7 @@ def main(args):
                         type=str,
                         help='Name to use for package name (when package is specified)')
     parser.add_argument('--arch',
-                        metavar='<amd64|i386|armhf|arm64|armel|all>',
+                        metavar='<amd64|arm64|all>',
                         type=str,
                         default=get_system_arch(),
                         help='Target architecture for build output')

build.sh

@@ -12,21 +12,24 @@ BUILD_NUM=${BUILD_NUM-$RANDOM}
 # Home dir of the docker user
 HOME_DIR=/root
 
+GO_VERSION=1.15.10
+
 imagename=kapacitor-builder-img-$BUILD_NUM
 dataname=kapacitor-data-$BUILD_NUM
 
 # Build new docker image
-docker build -f Dockerfile_build_ubuntu64 -t $imagename $DIR
+docker build -f Dockerfile_build_ubuntu64 -t $imagename --build-arg GO_VERSION=${GO_VERSION} $DIR
 
 # Build new docker image
-docker build -f Dockerfile_build_ubuntu64 -t influxdata/kapacitor-builder $DIR
+docker build -f Dockerfile_build_ubuntu64 -t influxdata/kapacitor-builder --build-arg GO_VERSION=${GO_VERSION} $DIR
 
 # Create data volume with code
 docker create \
     --name $dataname \
-    -v "$HOME_DIR/go/src/github.com/influxdata/kapacitor" \
+    -v "/go/src/github.com/influxdata/kapacitor" \
     $imagename /bin/true
-docker cp "$DIR/" "$dataname:$HOME_DIR/go/src/github.com/influxdata/"
+
+docker cp "$DIR/" "$dataname:/go/src/github.com/influxdata/"
 
 echo "Running build.py"
 # Run docker
@@ -38,6 +41,6 @@ docker run \
     $imagename \
     "$@"
 
-docker cp "$dataname:$HOME_DIR/go/src/github.com/influxdata/kapacitor/build" \
+docker cp "$dataname:/go/src/github.com/influxdata/kapacitor/build" \
     ./
 docker rm -v $dataname

go.mod

@@ -39,6 +39,7 @@ require (
 	github.com/influxdata/influx-cli/v2 v2.0.0-20210526124422-63da8eccbdb7
 	github.com/influxdata/influxdb v1.8.4
 	github.com/influxdata/influxdb/v2 v2.0.1-alpha.10.0.20210507184756-dc72dc3f0c07
+	github.com/influxdata/pkg-config v0.2.7
 	github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368
 	github.com/influxdata/wlog v0.0.0-20160411224016-7c63b0a71ef8
 	github.com/k-sone/snmpgo v3.2.0+incompatible

test.sh

@@ -34,6 +34,10 @@ TIMEOUT=${TIMEOUT-1000s}
 NO_UNCOMMITTED=${NO_UNCOMMITTED-false}
 # Home dir of the docker user
 HOME_DIR=/root
+# Go version
+GO_VERSION=1.15.10
+# GOPATH
+GOPATH=/go
 
 no_uncomitted_arg="$no_uncommitted_arg"
 if [ ! $NO_UNCOMMITTED ]
@@ -66,16 +70,16 @@ function run_test_docker {
     dataname="kapacitor-data-$BUILD_NUM"
 
     echo "Building docker image $imagename"
-    docker build -f "$dockerfile" -t "$imagename" .
+    docker build -f "$dockerfile" --build-arg GO_VERSION=$GO_VERSION -t "$imagename" .
 
     echo "Running test in docker $name with args $@"
 
     # Create data volume with code
     docker create \
         --name $dataname \
-        -v "$HOME_DIR/go/src/github.com/influxdata/kapacitor" \
+        -v "$GOPATH/src/github.com/influxdata/kapacitor" \
         $imagename /bin/true
-    docker cp "$DIR/" "$dataname:$HOME_DIR/go/src/github.com/influxdata/"
+    docker cp "$DIR/" "$dataname:$GOPATH/src/github.com/influxdata/"
 
     # Run tests in docker
     docker run \
@@ -92,7 +96,7 @@ function run_test_docker {
 
     # Copy results back out
     docker cp \
-        "$dataname:$HOME_DIR/go/src/github.com/influxdata/kapacitor/build" \
+        "$dataname:$GOPATH/src/github.com/influxdata/kapacitor/build" \
         ./
 
     # Remove the data and builder containers

tools.go

@@ -8,5 +8,6 @@ package kapacitor
 import (
 	_ "github.com/benbjohnson/tmpl"
 	_ "github.com/golang/protobuf/protoc-gen-go"
+	_ "github.com/influxdata/pkg-config"
 	_ "github.com/mailru/easyjson/easyjson"
 )