diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b02acce..679d2472 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,39 +1,51 @@ -version: 2 -jobs: - build: +--- +version: 2.1 +executors: + executor_med: # 2cpu, 4G ram docker: - - image: circleci/openjdk:11.0.2-jdk-stretch + - image: circleci/openjdk:11.0.4-jdk-stretch - image: postgres:11.4-alpine environment: POSTGRES_USER: test POSTGRES_PASSWORD: test POSTGRES_DB: orion - - working_directory: ~/repo - + resource_class: medium + working_directory: ~/project environment: - TERM: dumb JAVA_TOOL_OPTIONS: -Xmx2048m - GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=2 - GRADLE_MAX_TEST_FORKS: 2 + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=2 -Xmx2048m POSTGRES_USER: test POSTGRES_PASSWORD: test POSTGRES_DB: orion + executor_machine: + machine: + image: ubuntu-1604:201903-01 #Ubuntu 16.04, docker 18.09.3, docker-compose 1.23.1 + docker_layer_caching: true + working_directory: ~/project + +commands: + prepare: + description: "Prepare" steps: - checkout - - run: - name: Install Sodium Dependencies + name: Install Dependencies command: | - sudo apt-get install -y libsodium18 - + sudo apt-get update + sudo apt-get install -y libsodium18 libsodium-dev apt-transport-https - restore_cache: - name: Restoring cached gradle dependencies + name: Restore cached gradle dependencies keys: - - v1-dependencies-{{ checksum "build.gradle" }} - - v1-dependencies- + - deps-{{ checksum "build.gradle" }}-{{ .Branch }}-{{ .Revision }} + - deps-{{ checksum "build.gradle" }} + - deps- +jobs: + build: + executor: executor_med + steps: + - prepare - run: name: Spotless checks command: ./gradlew spotlesscheck @@ -120,11 +132,11 @@ jobs: - run: name: Collecting JavaDoc command: | - find . -type d -regex ".*/build/docs/javadoc" | while read dir; do - module=`echo $dir | sed -e 's/build\/docs\/javadoc//'` - mkdir -p ~/docs/"$module" - (cd "$dir" && tar c .) | (cd ~/docs/"$module" && tar x) - done + find . -type d -regex ".*/build/docs/javadoc" | while read dir; do + module=`echo $dir | sed -e 's/build\/docs\/javadoc//'` + mkdir -p ~/docs/"$module" + (cd "$dir" && tar c .) | (cd ~/docs/"$module" && tar x) + done - store_artifacts: name: Uploading JavaDoc @@ -133,22 +145,43 @@ jobs: - save_cache: name: Caching gradle dependencies + key: deps-{{ checksum "build.gradle" }}-{{ .Branch }}-{{ .Revision }} paths: - - .gradle - - ~/.gradle - key: v1-dependencies-{{ checksum "build.gradle" }}-{{ .Branch }}-{{ .BuildNum }} + - .gradle + - ~/.gradle - persist_to_workspace: - root: ~/repo + root: ~/project paths: - - build/version - - build/distributions/* + - ./ + + buildDocker: + executor: executor_machine + steps: + - prepare + - run: + name: Install Packages - Java 11 + command: | + sudo add-apt-repository -y ppa:openjdk-r/ppa + sudo apt update + sudo apt install -y openjdk-11-jdk + sudo update-java-alternatives -s java-1.11.0-openjdk-amd64 + - attach_workspace: + at: ~/project + - run: + name: hadoLint + command: | + docker run --rm -i hadolint/hadolint < docker/Dockerfile + - run: + name: build and test docker image + command: | + mkdir -p docker/reports + ./gradlew --no-daemon testDocker upload-distribution: - docker: - - image: circleci/openjdk:11.0.2-jdk-stretch + executor: executor_med steps: - - checkout + - prepare - attach_workspace: at: ~/project - run: @@ -168,14 +201,40 @@ jobs: ./gradlew deploy fi + publishDocker: + executor: executor_med + steps: + - prepare + - setup_remote_docker + - attach_workspace: + at: ~/project + - run: + name: Publish Docker + command: | + docker login --username "${DOCKER_USER}" --password "${DOCKER_PASSWORD}" + ./gradlew --no-daemon --parallel "-Pbranch=${CIRCLE_BRANCH}" dockerUpload + workflows: version: 2 pipeline: jobs: - build + - buildDocker: + requires: + - build - upload-distribution: + filters: + branches: + only: + - master + - /^release-.*/ requires: - build + - publishDocker: filters: branches: - only: master + only: + - master + - /^release-.*/ + requires: + - buildDocker diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 153a6e82..00000000 --- a/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM openjdk:8-jdk - -RUN apt-get -qq update && \ - apt-get -qq -y install libsodium-dev - -EXPOSE 8080 -EXPOSE 8888 - -ADD build/distributions/orion*.tar.gz /tmp -RUN mv /tmp/orion* /orion - -VOLUME /data - -CMD /orion/bin/orion /data/orion.conf \ No newline at end of file diff --git a/build.gradle b/build.gradle index 7342c91c..d5201db3 100644 --- a/build.gradle +++ b/build.gradle @@ -12,6 +12,10 @@ */ import net.ltgt.gradle.errorprone.CheckSeverity +import java.time.OffsetDateTime +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter + buildscript { repositories { jcenter() } @@ -357,7 +361,7 @@ distTar { delete fileTree(dir:'build/distributions', include: '*.tar.gz') } compression = Compression.GZIP - extension = 'tar.gz' + archiveExtension = 'tar.gz' } distZip { @@ -367,6 +371,107 @@ distZip { } } +// rename the top level dir from orion- to orion and this makes it really +// simple for use in docker +tasks.register("dockerDistUntar") { + dependsOn distTar + dependsOn distZip + def dockerBuildDir = "build/docker-orion/" + def distTarFile = distTar.outputs.files.singleFile + def distTarFileName = distTar.outputs.files.singleFile.name.replace(".tar.gz", "") + + doFirst { + new File(dockerBuildDir).mkdir() + copy { + from tarTree(distTarFile) + into(dockerBuildDir) + } + file("${dockerBuildDir}/${distTarFileName}").renameTo("${dockerBuildDir}/orion") + } +} + +task distDocker(type: Exec) { + dependsOn dockerDistUntar + def dockerBuildVersion = project.hasProperty('release.releaseVersion') ? project.property('release.releaseVersion') : "${rootProject.version}" + def imageName = "pegasyseng/orion" + def image = project.hasProperty('release.releaseVersion') ? "${imageName}:" + project.property('release.releaseVersion') : "${imageName}:${project.version}" + def dockerBuildDir = "build/docker-orion/" + workingDir "${dockerBuildDir}" + + doFirst { + copy { + from file("${projectDir}/docker/Dockerfile") + into(workingDir) + } + } + + executable "sh" + args "-c", "docker build --build-arg BUILD_DATE=${buildTime()} --build-arg VERSION=${dockerBuildVersion} --build-arg VCS_REF=${getCheckedOutGitCommitHash()} -t ${image} ." +} + +task testDocker(type: Exec) { + dependsOn distDocker + def dockerReportsDir = "docker/reports/" + def imageName = "pegasyseng/orion" + def image = project.hasProperty('release.releaseVersion') ? "${imageName}:" + project.property('release.releaseVersion') : "${imageName}:${project.version}" + workingDir "docker" + + doFirst { + new File(dockerReportsDir).mkdir() + } + + executable "sh" + args "-c", "bash test.sh ${image}" +} + +task dockerUpload(type: Exec) { + dependsOn distDocker + def imageName = "pegasyseng/orion" + def image = project.hasProperty('release.releaseVersion') ? "${imageName}:" + project.property('release.releaseVersion') : "${imageName}:${project.version}" + def cmd = "docker push '${image}'" + def additionalTags = [] + + if (project.hasProperty('branch') && project.property('branch') == 'master') { + additionalTags.add('develop') + } + + if (! version ==~ /.*-SNAPSHOT/) { + additionalTags.add('latest') + additionalTags.add(version.split(/\./)[0..1].join('.')) + } + + additionalTags.each { tag -> cmd += " && docker tag '${image}' '${imageName}:${tag.trim()}' && docker push '${imageName}:${tag.trim()}'" } + executable "sh" + args "-c", cmd +} + +// http://label-schema.org/rc1/ +// using the RFC3339 format "2016-04-12T23:20:50.52Z" +def buildTime() { + return OffsetDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SS'Z'")) +} + +def getCheckedOutGitCommitHash() { + def gitFolder = "$projectDir/.git/" + if (!file(gitFolder).isDirectory()) { + // We are in a submodule. The file's contents are `gitdir: \n`. + // Read the file, cut off the front, and trim the whitespace. + gitFolder = file(gitFolder).text.substring(8).trim() + "/" + } + def takeFromHash = 8 + /* + * '.git/HEAD' contains either + * in case of detached head: the currently checked out commit hash + * otherwise: a reference to a file containing the current commit hash + */ + def head = new File(gitFolder + "HEAD").text.split(":") // .git/HEAD + def isCommit = head.length == 1 // e5a7c79edabbf7dd39888442df081b1c9d8e88fd + + if(isCommit) return head[0].trim().take(takeFromHash) // e5a7c79edabb + + def refHead = new File(gitFolder + head[1].trim()) // .git/refs/heads/master + refHead.text.trim().take takeFromHash +} ////// // Runtime diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 00000000..27a3afbb --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1 @@ +reports diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..ab7e2fba --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,31 @@ +FROM openjdk:11.0.2-jre-slim-stretch + +COPY orion /opt/orion/ +WORKDIR /opt/orion + +# Install libsodium +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends libsodium18=1.0.11-2; \ + apt-get clean; \ + rm -rf /var/lib/apt/lists/* + +# Expose services ports +# 8080 Node Port, 8888 Client Port +EXPOSE 8080 8888 + +ENTRYPOINT ["/opt/orion/bin/orion"] + +# Build-time metadata as defined at http://label-schema.org +ARG BUILD_DATE +ARG VCS_REF +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="Orion" \ + org.label-schema.description="Private Transaction Manager" \ + org.label-schema.url="https://docs.orion.pegasys.tech/" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vcs-url="https://github.com/PegaSysEng/orion.git" \ + org.label-schema.vendor="PegaSys" \ + org.label-schema.version=$VERSION \ + org.label-schema.schema-version="1.0" \ No newline at end of file diff --git a/docker/test.sh b/docker/test.sh new file mode 100755 index 00000000..12000b7a --- /dev/null +++ b/docker/test.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +export GOSS_PATH=tests/goss-linux-amd64 +export GOSS_OPTS="$GOSS_OPTS --format junit" +export GOSS_FILES_STRATEGY=cp +DOCKER_IMAGE=$1 +DOCKER_FILE="${2:-$PWD/Dockerfile}" + +cleanup() { + set +e + echo "Cleaning temporary data directory" + rm -rf "$tmp_dir" +} + +i=0 + +tmp_dir=$(mktemp -d /tmp/tmp.XXXXXXXXXX) +chmod 777 "$tmp_dir" +trap 'ret=$?;cleanup;exit $ret' EXIT +echo "Temp Data Directory: ${tmp_dir}" +ls $tmp_dir +mkdir -p ./reports + +## Generate public/private key pair +# pipe password 123456 to orion +yes 123456 | docker run -i --rm --mount type=bind,source=$tmp_dir,target=/data $DOCKER_IMAGE -g /data/nodeKey + +# Create password file +cat << EOF > $tmp_dir/passwordFile +123456 +EOF + +## Create orion configuration file +cat << EOF > $tmp_dir/orion.conf +nodeurl = "http://127.0.0.1:8080/" +nodeport = 8080 +nodenetworkinterface = "0.0.0.0" +clienturl = "http://127.0.0.1:8888/" +clientport = 8888 +clientnetworkinterface = "0.0.0.0" +publickeys = ["/data/nodeKey.pub"] +privatekeys = ["/data/nodeKey.key"] +passwords = "/data/passwordFile" +workdir = "/data" +tls = "off" +EOF + +ls -l $tmp_dir/ + +# Test for normal startup with ports opened +GOSS_FILES_PATH=tests/01 \ +bash tests/dgoss run --mount type=bind,source=$tmp_dir,target=/data $DOCKER_IMAGE /data/orion.conf \ +> ./reports/01.xml || i=`expr $i + 1` + +exit $i diff --git a/docker/tests/01/goss.yaml b/docker/tests/01/goss.yaml new file mode 100644 index 00000000..7f29172b --- /dev/null +++ b/docker/tests/01/goss.yaml @@ -0,0 +1,20 @@ +file: + /opt/orion/bin/orion: + exists: true + mode: "0755" + owner: root + group: root + filetype: file + contains: [] +port: + tcp:8080: + listening: true + ip: + - 0.0.0.0 + tcp:8888: + listening: true + ip: + - 0.0.0.0 +process: + java: + running: true diff --git a/docker/tests/01/goss_wait.yaml b/docker/tests/01/goss_wait.yaml new file mode 100644 index 00000000..d7c9fc95 --- /dev/null +++ b/docker/tests/01/goss_wait.yaml @@ -0,0 +1,9 @@ +port: + tcp:8080: + listening: true + ip: + - 0.0.0.0 + tcp:8888: + listening: true + ip: + - 0.0.0.0 \ No newline at end of file diff --git a/docker/tests/dgoss b/docker/tests/dgoss new file mode 100755 index 00000000..e2e06174 --- /dev/null +++ b/docker/tests/dgoss @@ -0,0 +1,113 @@ +#!/bin/bash + +set -e + +USAGE="USAGE: $(basename "$0") [run|edit] " +GOSS_FILES_PATH="${GOSS_FILES_PATH:-.}" + +info() { + echo -e "INFO: $*" >&2; +} +error() { + echo -e "ERROR: $*" >&2; + exit 1; +} + +cleanup() { + set +e + { kill "$log_pid" && wait "$log_pid"; } 2> /dev/null + rm -rf "$tmp_dir" + if [[ $id ]];then + info "Deleting container" + docker rm -vf "$id" > /dev/null + fi +} + +run(){ + # Copy in goss + cp "${GOSS_PATH}" "$tmp_dir/goss" + chmod 755 "$tmp_dir/goss" + [[ -e "${GOSS_FILES_PATH}/goss.yaml" ]] && cp "${GOSS_FILES_PATH}/goss.yaml" "$tmp_dir" + [[ -e "${GOSS_FILES_PATH}/goss_wait.yaml" ]] && cp "${GOSS_FILES_PATH}/goss_wait.yaml" "$tmp_dir" + [[ ! -z "${GOSS_VARS}" ]] && [[ -e "${GOSS_FILES_PATH}/${GOSS_VARS}" ]] && cp "${GOSS_FILES_PATH}/${GOSS_VARS}" "$tmp_dir" + + # Switch between mount or cp files strategy + GOSS_FILES_STRATEGY=${GOSS_FILES_STRATEGY:="mount"} + case "$GOSS_FILES_STRATEGY" in + mount) + info "Starting docker container" + id=$(docker run -d -v "$tmp_dir:/goss:z" "${@:2}") + docker logs -f "$id" > "$tmp_dir/docker_output.log" 2>&1 & + ;; + cp) + info "Creating docker container" + id=$(docker create ${@:2}) + info "Copy goss files into container" + docker cp $tmp_dir/. $id:/goss + info "Starting docker container" + docker start $id > /dev/null + ;; + *) error "Wrong goss files strategy used! Correct options are \"mount\" or \"cp\"." + esac + + log_pid=$! + info "Container ID: ${id:0:8}" +} + +get_docker_file() { + if docker exec "$id" sh -c "test -e $1" > /dev/null;then + mkdir -p "${GOSS_FILES_PATH}" + info "Copied '$1' from container to '${GOSS_FILES_PATH}'" + docker cp "$id:$1" "${GOSS_FILES_PATH}" + fi +} + +# Main +tmp_dir=$(mktemp -d /tmp/tmp.XXXXXXXXXX) +chmod 777 "$tmp_dir" +trap 'ret=$?;cleanup;exit $ret' EXIT + +GOSS_PATH="${GOSS_PATH:-$(which goss 2> /dev/null || true)}" +[[ $GOSS_PATH ]] || { error "Couldn't find goss installation, please set GOSS_PATH to it"; } +[[ ${GOSS_OPTS+x} ]] || GOSS_OPTS="--color --format documentation" +[[ ${GOSS_WAIT_OPTS+x} ]] || GOSS_WAIT_OPTS="-r 30s -s 1s > /dev/null" +GOSS_SLEEP=${GOSS_SLEEP:-0.2} + +case "$1" in + run) + run "$@" + if [[ -e "${GOSS_FILES_PATH}/goss_wait.yaml" ]]; then + info "Found goss_wait.yaml, waiting for it to pass before running tests" + if [[ -z "${GOSS_VARS}" ]]; then + if ! docker exec "$id" sh -c "/goss/goss -g /goss/goss_wait.yaml validate $GOSS_WAIT_OPTS"; then + error "goss_wait.yaml never passed" + fi + else + if ! docker exec "$id" sh -c "/goss/goss -g /goss/goss_wait.yaml --vars='/goss/${GOSS_VARS}' validate $GOSS_WAIT_OPTS"; then + error "goss_wait.yaml never passed" + fi + fi + fi + [[ $GOSS_SLEEP ]] && { info "Sleeping for $GOSS_SLEEP"; sleep "$GOSS_SLEEP"; } + # info "Container health" + # if ! docker top $id; then + # docker logs $id + # fi + info "Running Tests" + if [[ -z "${GOSS_VARS}" ]]; then + docker exec "$id" sh -c "/goss/goss -g /goss/goss.yaml validate $GOSS_OPTS" + else + docker exec "$id" sh -c "/goss/goss -g /goss/goss.yaml --vars='/goss/${GOSS_VARS}' validate $GOSS_OPTS" + fi + ;; + edit) + run "$@" + info "Run goss add/autoadd to add resources" + docker exec -it "$id" sh -c 'cd /goss; PATH="/goss:$PATH" exec sh' + get_docker_file "/goss/goss.yaml" + get_docker_file "/goss/goss_wait.yaml" + [[ ! -z "${GOSS_VARS}" ]] && get_docker_file "/goss/${GOSS_VARS}" + ;; + *) + error "$USAGE" +esac diff --git a/docker/tests/goss-linux-amd64 b/docker/tests/goss-linux-amd64 new file mode 100755 index 00000000..62f2325a Binary files /dev/null and b/docker/tests/goss-linux-amd64 differ