Skip to content
This repository has been archived by the owner on Mar 3, 2023. It is now read-only.

Commit

Permalink
Java 11 support (#3454)
Browse files Browse the repository at this point in the history
* Support Java 11

* config travis to use oracle jdk 11

* Java 11 support (#3399)

* Support Java 11

* config travis to use oracle jdk 11

* Add check jdk version

* Fix command arguments.

Change insert gc_options

Update list

Fix gc-logging

* Add missing parameter

* typo

* Add pause time

* Add python3 for docker files

* Revert "config travis to use oracle jdk 11"

This reverts commit edaa84f.

* Update unitest

* Update unittest

* Add JDK11 UnitTest

* Add Unittest using Docker.

* Add full test

* Change JDK11

* Ignore PowerMock used for testing is not compatible with Java 11.

* Change JDK11 for travis.yml

* Update python

* Update library requests, pylint

* Revert "Update library requests, pylint"

This reverts commit 7706d34.

* Add ignore

* Refactoring ReflectionUtils

Co-authored-by: Ning Wang <wangninggm@gmail.com>
  • Loading branch information
thinker0 and nwangtw authored Feb 19, 2020
1 parent 4014a9c commit 62441d8
Show file tree
Hide file tree
Showing 61 changed files with 764 additions and 165 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ dist: trusty
language: java

jdk:
- oraclejdk8
- oraclejdk11

addons:
apt:
Expand Down
13 changes: 12 additions & 1 deletion WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ maven_jar(

maven_jar(
name = "javax_annotation_javax_annotation_api",
artifact = "javax.annotation:javax.annotation-api:1.2",
artifact = "javax.annotation:javax.annotation-api:1.3.2",
)

maven_jar(
Expand Down Expand Up @@ -735,6 +735,17 @@ maven_jar(
artifact = "org.apache.commons:commons-compress:1.14",
)

maven_jar(
name = "javax_xml_bind_jaxb",
artifact = "javax.xml.bind:jaxb-api:2.3.0",
)

maven_jar(
name = "javax_activation_activation",
artifact = "javax.activation:activation:1.1.1",
)


# bookkeeper & distributedlog dependencies
maven_jar(
name = "org_apache_distributedlog_core",
Expand Down
10 changes: 5 additions & 5 deletions docker/compile/Dockerfile.centos7
Original file line number Diff line number Diff line change
Expand Up @@ -35,18 +35,18 @@ RUN yum -y install \
libtool \
make \
patch \
python \
python-devel \
python-setuptools \
python3-devel \
python3-setuptools \
zip \
unzip \
wget \
which \
tree \
java-1.8.0-openjdk \
java-1.8.0-openjdk-devel
java-11-openjdk \
java-11-openjdk-devel

ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk

RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/$bazelVersion/bazel-$bazelVersion-installer-linux-x86_64.sh \
&& chmod +x /tmp/bazel.sh \
Expand Down
11 changes: 7 additions & 4 deletions docker/compile/Dockerfile.debian9
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,17 @@ RUN apt-get update && apt-get -y install \
git \
libtool \
libtool-bin \
python \
python2.7-dev \
python-dev \
python3-dev \
software-properties-common \
python-setuptools \
python3-setuptools \
tree \
zip \
unzip \
wget
wget \
openjdk-11-jdk-headless

ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64

RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/$bazelVersion/bazel-$bazelVersion-installer-linux-x86_64.sh \
&& chmod +x /tmp/bazel.sh \
Expand Down
12 changes: 6 additions & 6 deletions docker/compile/Dockerfile.ubuntu14.04
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,18 @@ RUN apt-get update && apt-get -y install \
libssl-dev \
git \
libtool \
python \
python2.7-dev \
python-software-properties \
python-setuptools \
python-dev \
python3-dev \
python3-software-properties \
python3-setuptools \
zip \
unzip \
wget \
cmake \
tree \
openjdk-8-jdk-headless
openjdk-11-jdk-headless

ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64

RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/$bazelVersion/bazel-$bazelVersion-installer-linux-x86_64.sh \
&& chmod +x /tmp/bazel.sh \
Expand Down
15 changes: 7 additions & 8 deletions docker/compile/Dockerfile.ubuntu16.04
Original file line number Diff line number Diff line change
Expand Up @@ -31,19 +31,18 @@ RUN apt-get update && apt-get -y install \
libtool-bin \
libunwind8 \
libunwind-setjmp0-dev \
python \
python2.7-dev \
python-software-properties \
python-dev \
python3-dev \
python3-software-properties \
software-properties-common \
python-setuptools \
python3-setuptools \
tree \
zip \
unzip \
wget
wget \
openjdk-11-jdk-headless

RUN apt-get update && apt-get -y install openjdk-8-jdk-headless

ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64

RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/$bazelVersion/bazel-$bazelVersion-installer-linux-x86_64.sh \
&& chmod +x /tmp/bazel.sh \
Expand Down
5 changes: 3 additions & 2 deletions docker/compile/Dockerfile.ubuntu18.04
Original file line number Diff line number Diff line change
Expand Up @@ -29,16 +29,17 @@ RUN apt-get update && apt-get -y install \
libunwind8 \
patch \
python-dev \
python3-dev \
wget \
zip \
virtualenv \
unzip \
git \
curl \
tree \
openjdk-8-jdk-headless
openjdk-11-jdk-headless

ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64

RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/$bazelVersion/bazel-$bazelVersion-installer-linux-x86_64.sh \
&& chmod +x /tmp/bazel.sh \
Expand Down
6 changes: 3 additions & 3 deletions docker/dist/Dockerfile.dist.centos7
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,13 @@ RUN yum -y install unzip; yum clean all
RUN yum -y install which; yum clean all
RUN yum -y install curl; yum clean all
RUN yum -y install nmap-ncat; yum clean all
RUN yum -y install python-setuptools; yum clean all
RUN yum -y install python python3-setuptools; yum clean all
RUN easy_install supervisor
RUN echo_supervisord_conf > /etc/supervisord.conf

RUN yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel; yum clean all
RUN yum -y install java-11-openjdk java-11-openjdk-devel; yum clean all

ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk

ADD artifacts /heron

Expand Down
4 changes: 3 additions & 1 deletion docker/dist/Dockerfile.dist.debian9
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,12 @@ FROM openjdk:8-jdk-slim
RUN apt-get -y update && apt-get -y install \
netcat-openbsd \
python \
python3 \
unzip \
curl \
vim \
supervisor && \
supervisor \
openjdk-11-jdk-headless && \
apt-get clean all && \
rm -rf /var/lib/apt/lists/*

Expand Down
5 changes: 3 additions & 2 deletions docker/dist/Dockerfile.dist.ubuntu14.04
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,16 @@ FROM ubuntu:14.04

RUN apt-get -y update && apt-get -y install \
python \
python3 \
unzip \
software-properties-common \
supervisor \
curl

RUN add-apt-repository ppa:openjdk-r/ppa && apt-get -y update && \
apt-get -y install openjdk-8-jdk
apt-get -y install openjdk-11-jdk-headless

ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
RUN update-ca-certificates -f

ADD artifacts /heron
Expand Down
5 changes: 3 additions & 2 deletions docker/dist/Dockerfile.dist.ubuntu16.04
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,14 @@ RUN apt-get update

RUN apt-get -y install \
python \
python3 \
unzip \
software-properties-common \
curl \
supervisor \
openjdk-8-jdk-headless
openjdk-11-jdk-headless

ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
RUN update-ca-certificates -f

ADD artifacts /heron
Expand Down
4 changes: 2 additions & 2 deletions docker/dist/Dockerfile.dist.ubuntu18.04
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ FROM ubuntu:18.04

RUN apt-get update
RUN apt-get -y install \
unzip software-properties-common curl supervisor openjdk-8-jdk-headless
unzip software-properties-common curl python python3 supervisor openjdk-11-jdk-headless

ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
RUN update-ca-certificates -f

ADD artifacts /heron
Expand Down
64 changes: 64 additions & 0 deletions docker/scripts/test-docker.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -o nounset
set -o errexit

realpath() {
echo "$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
}

DOCKER_DIR=$(dirname $(dirname $(realpath $0)))
PROJECT_DIR=$(dirname $DOCKER_DIR)

verify_dockerfile_exists() {
if [ ! -f $1 ]; then
echo "The Dockerfiler $1 does not exist"
exit 1
fi
}

dockerfile_path_for_platform() {
echo "$SCRATCH_DIR/test/Dockerfile.$1"
}

copy_bazel_rc_to() {
cp $PROJECT_DIR/tools/docker/bazel.rc $1
}

DOCKER_FILE=$(dockerfile_path_for_platform $TARGET_PLATFORM)
verify_dockerfile_exists $DOCKER_FILE
copy_bazel_rc_to $SCRATCH_DIR/bazelrc

echo "Building heron-compiler container"
docker build -t heron-compiler:$TARGET_PLATFORM -f $DOCKER_FILE $SCRATCH_DIR

echo "Running build in container"
docker run \
--rm \
-e TARGET_PLATFORM=$TARGET_PLATFORM \
-e SCRATCH_DIR="/scratch" \
-e SOURCE_TARBALL="/src.tar.gz" \
-e HERON_VERSION=$HERON_VERSION \
-e HERON_GIT_REV="${HERON_GIT_REV}" \
-e HERON_BUILD_VERSION="${HERON_BUILD_VERSION}" \
-e HERON_BUILD_HOST="${HERON_BUILD_HOST}" \
-e HERON_BUILD_USER="${HERON_BUILD_USER}" \
-e HERON_BUILD_TIME="${HERON_BUILD_TIME}" \
-e HERON_TREE_STATUS="${HERON_TREE_STATUS}" \
-v "$SOURCE_TARBALL:/src.tar.gz:ro" \
-t heron-compiler:$TARGET_PLATFORM /test-platform.sh
76 changes: 76 additions & 0 deletions docker/scripts/test-platform.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -o nounset
set -o errexit

# By default bazel creates files with mode 0555 which means they are only able to be read and not written to. This
# causes cp to fail when trying to overwrite the file. This makes sure that if the file exists we can overwrite it.
function copyFileToDest() {
if [ -f $2 ]; then
chmod 755 $2
fi

cp $1 $2
}

echo "Building heron with version $HERON_VERSION for platform $TARGET_PLATFORM"

mkdir -p $SCRATCH_DIR
cd $SCRATCH_DIR

echo "Extracting source"
tar -C . -xzf $SOURCE_TARBALL

if [[ "$TARGET_PLATFORM" =~ "ubuntu" ]]; then
CONFIG_PLATFORM=ubuntu
elif [[ "$TARGET_PLATFORM" =~ "centos" ]]; then
CONFIG_PLATFORM=centos
elif [[ "$TARGET_PLATFORM" =~ "darwin" ]]; then
CONFIG_PLATFORM=darwin
elif [[ "$TARGET_PLATFORM" =~ "debian" ]]; then
CONFIG_PLATFORM=debian
elif [[ "$TARGET_PLATFORM" =~ "ubuntu_nostyle" ]]; then
CONFIG_PLATFORM=ubuntu
elif [[ "$TARGET_PLATFORM" =~ "centos_nostyle" ]]; then
CONFIG_PLATFORM=centos
elif [[ "$TARGET_PLATFORM" =~ "darwin_nostyle" ]]; then
CONFIG_PLATFORM=darwin
elif [[ "$TARGET_PLATFORM" =~ "debian_nostyle" ]]; then
CONFIG_PLATFORM=debian
else
echo "Unknown platform: $TARGET_PLATFORM"
exit 1
fi

bazel version
./bazel_configure.py
bazel clean

echo "UnitTest"
bazel test -c opt --jobs 25 \
--config=$CONFIG_PLATFORM \
--test_output=all \
--test_summary=detailed \
heron/... \
heronpy/... \
examples/... \
storm-compatibility-examples/... \
eco-storm-examples/... \
eco-heron-examples/... \
contrib/...

Loading

0 comments on commit 62441d8

Please sign in to comment.