Java 11 support - backport to 6x

.ruby-version

@@ -1 +1 @@
-jruby-9.1.12.0
+jruby-9.2.4.1

Dockerfile

@@ -1,6 +1,24 @@
-FROM container-registry-test.elastic.co/logstash-test/logstash-base:latest
+FROM ubuntu:xenial
 
-RUN ln -s /tmp/vendor /opt/logstash/vendor
+RUN apt-get update && \
+    apt-get install -y zlib1g-dev build-essential vim rake git curl libssl-dev libreadline-dev libyaml-dev  \
+      libxml2-dev libxslt-dev openjdk-8-jdk-headless curl iputils-ping netcat && \
+    apt-get clean
+
+WORKDIR /root
+
+RUN adduser --disabled-password --gecos "" --home /home/logstash logstash && \
+    mkdir -p /usr/local/share/ruby-build && \
+    mkdir -p /opt/logstash && \
+    mkdir -p /opt/logstash/data && \
+    mkdir -p /mnt/host && \
+    chown logstash:logstash /opt/logstash
+
+USER logstash
+WORKDIR /home/logstash
+
+# used by the purge policy
+LABEL retention="keep"
 
 ADD gradlew /opt/logstash/gradlew
 ADD gradle/wrapper /opt/logstash/gradle/wrapper

Dockerfile.base

@@ -16,30 +16,7 @@ RUN adduser --disabled-password --gecos "" --home /home/logstash logstash && \
 
 USER logstash
 WORKDIR /home/logstash
-
-RUN git clone https://github.com/sstephenson/rbenv.git .rbenv && \
-    git clone https://github.com/sstephenson/ruby-build.git .rbenv/plugins/ruby-build && \
-    echo 'export PATH=/home/logstash/.rbenv/bin:$PATH' >> /home/logstash/.bashrc
-
-ENV PATH "/home/logstash/.rbenv/bin:$PATH"
-
-#Only used to help bootstrap the build (not to run Logstash itself)
-RUN echo 'eval "$(rbenv init -)"' >> .bashrc && \
-    rbenv install jruby-9.1.12.0 && \
-    rbenv global jruby-9.1.12.0 && \
-    bash -i -c 'gem install bundler' && \
-    rbenv local jruby-9.1.12.0 && \
-    mkdir -p /opt/logstash/data
-
-
-# Create a cache for the dependencies based on the current master, any dependencies not cached will be downloaded at runtime
-RUN git clone https://github.com/elastic/logstash.git /tmp/logstash && \
-    cd /tmp/logstash && \
-    ./gradlew bootstrap compileJava compileTestJava && \
-    cd qa/integration && \
-    /home/logstash/.rbenv/shims/bundle install && \
-    mv /tmp/logstash/vendor /tmp/vendor && \
-    rm -rf /tmp/logstash
+RUN mkdir -p /opt/logstash/data
 
 # used by the purge policy
-LABEL retention="keep"
+LABEL retention="keep"

Gemfile.template

@@ -21,11 +21,12 @@ gem "rack-test", :require => "rack/test", :group => :development
 gem "flores", "~> 0.0.6", :group => :development
 gem "term-ansicolor", "~> 1.3.2", :group => :development
 gem "json-schema", "~> 2.6", :group => :development
+gem "webmock", "~> 3.5.1", :group => :development
 gem "belzebuth", :group => :development
 gem "pleaserun", "~>0.0.28"
 gem 'webrick', '~> 1.3.1'
 gem "atomic", "<= 1.1.99"
-gem "rake", "~> 12.2.1", :group => :build
+gem "rake", "~> 12.2.1"
 gem "logstash-codec-cef"
 gem "logstash-codec-collectd"
 gem "logstash-codec-dots"
@@ -115,7 +116,6 @@ gem "logstash-output-kafka"
 gem "logstash-output-lumberjack"
 gem "logstash-output-nagios"
 gem "logstash-output-null"
-gem "logstash-output-pagerduty"
 gem "logstash-output-pipe"
 gem "logstash-output-rabbitmq"
 gem "logstash-output-redis"

bin/logstash.lib.sh

@@ -137,7 +137,7 @@ setup_vendored_jruby() {
   fi
 
   if [ -z "$LS_GEM_HOME" ] ; then
-    export GEM_HOME="${LOGSTASH_HOME}/vendor/bundle/jruby/2.3.0"
+    export GEM_HOME="${LOGSTASH_HOME}/vendor/bundle/jruby/2.5.0"
   else
     export GEM_HOME=${LS_GEM_HOME}
   fi

build.gradle

@@ -57,6 +57,7 @@ allprojects {
       exceptionFormat "full"
       showCauses true
       showStackTraces true
+      enableAssertions false
 
       // set options for log level DEBUG and INFO
       debug {
@@ -171,7 +172,7 @@ verifyFile.onlyIf { customJRubyDir == "" }
 
 task buildCustomJRuby(type: Exec) {
   description "Build tar.gz and .jar artifacts from JRuby source directory"
-  workingDir customJRubyDir
+  workingDir (customJRubyDir == "" ? "./" : customJRubyDir)
   commandLine './mvnw', 'clean', 'install', '-Pdist', '-Pcomplete'
   standardOutput = new ByteArrayOutputStream()
   errorOutput = new ByteArrayOutputStream()
@@ -208,6 +209,9 @@ task downloadAndInstallJRuby(dependsOn: [verifyFile, installCustomJRuby], type:
     exclude "**/stdlib/rdoc/**"
     includeEmptyDirs = false
     into "${projectDir}/vendor/jruby"
+    doLast {
+      rubyGradleUtils.gem("rake", "12.3.1", "${projectDir}/vendor/bundle/jruby/2.5.0")
+    }
 }
 
 downloadAndInstallJRuby.onlyIf { customJRubyDir == "" }
@@ -219,8 +223,9 @@ task installDefaultGems(dependsOn: downloadAndInstallJRuby) {
   outputs.file("${projectDir}/Gemfile")
   outputs.file("${projectDir}/Gemfile.lock")
   outputs.dir("${projectDir}/logstash-core/lib/jars")
-  outputs.dir("${projectDir}/vendor/bundle/jruby/2.3.0")
+  outputs.dir("${projectDir}/vendor/bundle/jruby/2.5.0")
   doLast {
+    rubyGradleUtils.gem("rake", "12.3.1", "${projectDir}/vendor/bundle/jruby/2.5.0")
     rubyGradleUtils.rake('plugin:install-default')
   }
 }
@@ -236,8 +241,9 @@ task installTestGems(dependsOn: assemblyDeps) {
   outputs.file("${projectDir}/Gemfile")
   outputs.file("${projectDir}/Gemfile.lock")
   outputs.dir("${projectDir}/logstash-core/lib/jars")
-  outputs.dir("${projectDir}/vendor/bundle/jruby/2.3.0")
+  outputs.dir("${projectDir}/vendor/bundle/jruby/2.5.0")
   doLast {
+    rubyGradleUtils.gem("rake", "12.3.1", "${projectDir}/vendor/bundle/jruby/2.5.0")
     rubyGradleUtils.rake('test:install-core')
   }
 }
@@ -254,6 +260,7 @@ task assembleTarDistribution(dependsOn: assemblyDeps) {
   inputs.files fileTree("${projectDir}/x-pack")
   outputs.files file("${buildDir}/logstash-${project.version}-SNAPSHOT.tar.gz")
   doLast {
+    rubyGradleUtils.gem("rake", "12.3.1", "${projectDir}/vendor/bundle/jruby/2.5.0")
     rubyGradleUtils.rake('artifact:tar')
   }
 }
@@ -314,13 +321,13 @@ task unpackTarDistribution(dependsOn: assembleTarDistribution, type: Copy) {
 }
 
 def qaVendorPath = "${buildDir}/qa/integration/vendor"
-def qaBundledGemPath = "${qaVendorPath}/jruby/2.3.0"
+def qaBundledGemPath = "${qaVendorPath}/jruby/2.5.0"
 def qaBundleBin = "${qaBundledGemPath}/bin/bundle"
 
 task installIntegrationTestBundler(dependsOn: unpackTarDistribution) {
-  outputs.files fileTree("${qaBundledGemPath}/gems/bundler-1.16.0")
+  outputs.files fileTree("${qaBundledGemPath}/gems/bundler-1.17.1")
   doLast {
-    rubyGradleUtils.gem("bundler", "1.16.0", qaBundledGemPath)
+    rubyGradleUtils.gem("bundler", "1.17.1", qaBundledGemPath)
   }
 }
 

buildSrc/build.gradle

@@ -8,5 +8,5 @@ repositories {
 }
 
 dependencies {
-  compile group: 'org.jruby', name: 'jruby-complete', version: '9.1.13.0'
+  compile group: 'org.jruby', name: 'jruby-complete', version: '9.2.5.0'
 }

buildSrc/src/main/groovy/org/logstash/gradle/RubyGradleUtils.groovy

@@ -88,7 +88,7 @@ final class RubyGradleUtils {
   Object executeJruby(Closure<?> block) {
     def jruby = new ScriptingContainer()
     def env = jruby.environment
-    def gemDir = "${projectDir}/vendor/bundle/jruby/2.3.0".toString()
+    def gemDir = "${projectDir}/vendor/bundle/jruby/2.5.0".toString()
     env.put "USE_RUBY", "1"
     env.put "GEM_HOME", gemDir
     env.put "GEM_SPEC_CACHE", "${buildDir}/cache".toString()

config/jvm.options

@@ -17,7 +17,6 @@
 ################################################################
 
 ## GC configuration
--XX:+UseParNewGC
 -XX:+UseConcMarkSweepGC
 -XX:CMSInitiatingOccupancyFraction=75
 -XX:+UseCMSInitiatingOccupancyOnly

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.10-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.3-all.zip

gradlew

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 ##############################################################################
 ##
@@ -33,11 +33,11 @@ DEFAULT_JVM_OPTS=""
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
-warn ( ) {
+warn () {
     echo "$*"
 }
 
-die ( ) {
+die () {
     echo
     echo "$*"
     echo
@@ -154,11 +154,19 @@ if $cygwin ; then
     esac
 fi
 
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
 }
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+APP_ARGS=$(save "$@")
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
+
+exec "$JAVACMD" "$@"

gradlew.bat

@@ -49,7 +49,6 @@ goto fail
 @rem Get command-line arguments, handling Windows variants
 
 if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
 
 :win9xME_args
 @rem Slurp the command line arguments.
@@ -60,11 +59,6 @@ set _SKIP=2
 if "x%~1" == "x" goto execute
 
 set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
 
 :execute
 @rem Setup the command line

lib/bootstrap/bundler.rb

@@ -10,15 +10,20 @@ def patch!
       # Patch to prevent Bundler to save a .bundle/config file in the root
       # of the application
       ::Bundler::Settings.module_exec do
-        def set_key(key, value, hash, file)
-          key = key_for(key)
-
-          unless hash[key] == value
-            hash[key] = value
-            hash.delete(key) if value.nil?
-          end
+        def set_local(key, value)
+          set_key(key, value, @local_config, nil)
+        end
+      end
 
-          value
+      # In recent versions (currently 1.17.1) Bundler calls reset_paths! early during
+      # Bundler::CLI.start (https://github.com/bundler/bundler/blob/v1.17.1/lib/bundler/cli.rb#L39)
+      # This breaks our setting up of gemfile and bundle paths, the without group setting etc
+      # We need to tone down this very aggressive resetter (https://github.com/bundler/bundler/blob/v1.17.1/lib/bundler.rb#L487-L500)
+      # So we reimplement it here to only nullify the definition object, so that it can be computed
+      # again if necessary with all the configuration in place.
+      ::Bundler.module_exec do
+        def self.reset_paths!
+          @definition = nil
         end
       end
 
@@ -47,11 +52,11 @@ def setup!(options = {})
       require "bundler"
       LogStash::Bundler.patch!
 
-      ::Bundler.settings[:path] = Environment::BUNDLE_DIR
-      ::Bundler.settings[:without] = options[:without].join(":")
+      ::Bundler.settings.set_local(:path, Environment::BUNDLE_DIR)
+      ::Bundler.settings.set_local(:without, options[:without])
       # in the context of Bundler.setup it looks like this is useless here because Gemfile path can only be specified using
       # the ENV, see https://github.com/bundler/bundler/blob/v1.8.3/lib/bundler/shared_helpers.rb#L103
-      ::Bundler.settings[:gemfile] = Environment::GEMFILE_PATH
+      ::Bundler.settings.set_local(:gemfile, Environment::GEMFILE_PATH)
 
       ::Bundler.reset!
       ::Bundler.setup
@@ -93,7 +98,7 @@ def invoke!(options = {})
         )
       end
       # create Gemfile.jruby-1.9.lock from template iff a template exists it itself does not exist
-      lock_template = ::File.join(ENV["LOGSTASH_HOME"], "Gemfile.jruby-2.3.lock.release")
+      lock_template = ::File.join(ENV["LOGSTASH_HOME"], "Gemfile.jruby-2.5.lock.release")
       if ::File.exists?(lock_template) && !::File.exists?(Environment::LOCKFILE)
         FileUtils.copy(lock_template, Environment::LOCKFILE)
       end
@@ -103,10 +108,10 @@ def invoke!(options = {})
       # force Rubygems sources to our Gemfile sources
       ::Gem.sources = ::Gem::SourceList.from(options[:rubygems_source]) if options[:rubygems_source]
 
-      ::Bundler.settings[:path] = LogStash::Environment::BUNDLE_DIR
-      ::Bundler.settings[:gemfile] = LogStash::Environment::GEMFILE_PATH
-      ::Bundler.settings[:without] = options[:without].join(":")
-      ::Bundler.settings[:force] = options[:force]
+      ::Bundler.settings.set_local(:path, LogStash::Environment::BUNDLE_DIR)
+      ::Bundler.settings.set_local(:gemfile, LogStash::Environment::GEMFILE_PATH)
+      ::Bundler.settings.set_local(:without, options[:without])
+      ::Bundler.settings.set_local(:force, options[:force])
 
       if !debug?
         # Will deal with transient network errors

lib/bootstrap/rspec.rb

@@ -10,6 +10,8 @@
 require "rspec"
 require 'ci/reporter/rake/rspec_loader'
 
+RSpec.world.reset # if multiple rspec runs occur in a single process, the RSpec "world" state needs to be reset.
+
 status = RSpec::Core::Runner.run(ARGV.empty? ? ($JUNIT_ARGV || ["spec"]) : ARGV).to_i
 if ENV["IS_JUNIT_RUN"]
   return status