[JENKINS-52693] - External Build Logging. Reference Implementation for Elasticsearch

Jenkinsfile

@@ -0,0 +1 @@
+buildPlugin(platforms: ['linux'])

demo/Makefile

@@ -0,0 +1,33 @@
+# Just a Makefile for manual testing
+.PHONY: all
+
+ARTIFACT_ID = jenkins-external-task-logging-elk-demo
+VERSION = 2.131-elk-SNAPSHOT
+CWP_VERSION= 1.0
+
+all: clean build
+
+clean:
+	rm -rf tmp
+	mkdir tmp
+
+build: tmp/output/target/${ARTIFACT_ID}-${VERSION}.war
+
+tmp/output/target/${ARTIFACT_ID}-${VERSION}.war:
+	mvn com.googlecode.maven-download-plugin:download-maven-plugin:1.4.0:artifact \
+		-DgroupId=io.jenkins.tools.custom-war-packager \
+		-DartifactId=custom-war-packager-cli \
+		-Dclassifier=jar-with-dependencies \
+		-Dversion=${CWP_VERSION} \
+		-DoutputDirectory=tmp \
+		-DoutputFileName=cwp-cli.jar
+	java -jar tmp/cwp-cli.jar \
+	    -configPath packager-config.yml -version ${VERSION}
+
+run: tmp/output/target/${ARTIFACT_ID}-${VERSION}.war
+	docker-compose rm -fv
+	docker-compose up --build --force-recreate jenkins elk
+
+debug: tmp/output/target/${ARTIFACT_ID}-${VERSION}.war
+	docker-compose rm -fv
+	docker-compose up --build --force-recreate jenkinsDebug elk

demo/README.md

@@ -0,0 +1,72 @@
+External Task Logging to Elasticsearch Demo
+===
+
+This demo packages Jenkins WAR for External Task Logging to Elasticsearch with help of Logstash plugin.
+
+This demo includes [Logstash Plugin PR#18](https://github.com/jenkinsci/logstash-plugin/pull/18) and
+all its upstream dependencies. 
+It also bundles auto-configuration System Groovy scripts, so that the WAR file starts
+up with pre-configured Logstash plugin settings and some other configs.
+
+Features of the demo:
+
+* Pipeline jobs logging goes to Elasticsearch
+* When tasks are executed on agents, the logs get posted to Elasticsearch directly
+  without passing though the master and causing scalability issues
+* Pipeline jobs override standard Log actions in the Jenkins core, so the
+  underlying implementation is transparent to users
+* Secrets are escaped in stored/displayed logs when running on master and agents.
+* Console annotations work as they work for common Jenkins instances
+* Log blocks are collapsible in the _Console_ screen
+* Origin container ID of every message is visible in Kibana (if you have set that up) via sender field
+
+The demo can be run in Docker Compose,
+ELK stack is provided by the [sebp/elk](https://hub.docker.com/r/sebp/elk/)  image in this case.
+
+## Prerequisites
+
+* Docker and Docker Compose are installed
+
+## Building demo
+
+To build the demo...
+
+1. Go to the repository root, run `mvn clean package` to build Jenkins Custom WAR Packager
+2. Change directory to the demo root
+3. Run `make build`
+
+First build may take a while, because the packager will need to checkout and build 
+many repositories.
+
+## Running demo
+
+1. Run `make run`. It will spin up the demo with predefined environment.
+   Jenkins will be available on the port 8080, credentials: `admin/admin`
+2. If you want to run demo jobs on the agent, 
+also run `docker-compose up agent` in a separate terminal window
+3. In order to access the instance, use the "admin/admin" credentials.
+4. Run one of the demo jobs.   
+5. Browse logs
+  * Classic Log action queries data from Elasticsearch
+  * There is a _Log (Kibana)_ action in runs, which shows Kibana. 
+  * In order to see Kibana logs, you will need to configure the default index in the 
+    embedded page once Jenkins starts up. Use `logstash/` as a default index and 
+    `@timestamp` as data source
+
+## Manual run
+
+This guideline allows to run the demo locally.
+Only Logstash will be preconfigured.
+
+1. Run `docker run -p 5601:5601 -p 9200:9200 -p 5044:5044 -it --name elk sebp/elk:es241_l240_k461` 
+to start the Docker container to to expose ports
+2. Run Jenkins using `JENKINS_HOME=$(pwd)/work java -jar tmp/output/target/external-task-logging-elk-2.107.3-elk-SNAPSHOT.war --httpPort=8080 --prefix=/jenkins` 
+(or just `run run.sh`).
+  * If needed, the demo can be configured by setting system properties
+  * `elasticsearch.host` - host, defaults to `http://elk`
+  * `elasticsearch.port` - Elasticsearch port, defaults to `9200`
+  * `logstash.key` - Path to the root index/key for logging, defaults to `/logstash/logs`
+  * `elasticsearch.username` and `elasticsearch.password` - 
+3. Pass through the installation Wizard
+4. Create a Pipeline job with some logging (e.g. `echo` commands), run it
+5. Browse logs (see above)

demo/docker-compose.yml

@@ -0,0 +1,51 @@
+version: '3'
+
+services:
+  elk:
+  #TODO: update to ES 5.0
+    image: sebp/elk:es241_l240_k461
+    container_name: elk
+    ports:
+      - "5601:5601" # Kibana
+      - "9200:9200" # Elasticsearch
+      - "5044:5044" # Logstash
+    expose:
+      - "5601"
+      - "9200"
+      - "5044"
+  jenkins:
+    image: jenkins/demo-external-task-logging-elk:latest
+    container_name: jenkins
+    links:
+      - elk
+    environment:
+      - JAVA_OPTS=-Dio.jenkins.demo.external-task-logging-elk.enabled=true -Djenkins.install.runSetupWizard=false
+    ports:
+      - "8080:8080"
+      - "9000:9000"
+    expose:
+      - "8080"
+      - "9000"
+  jenkinsDebug:
+    image: jenkins/demo-external-task-logging-elk:latest
+    container_name: jenkins
+    links:
+      - elk
+    environment:
+      - JAVA_OPTS=-Dio.jenkins.demo.external-task-logging-elk.enabled=true -Djenkins.install.runSetupWizard=false
+      - DEBUG=true
+    ports:
+      - "8080:8080"
+      - "9000:9000"
+      - "5005:5005"
+    expose:
+      - "8080"
+      - "9000"
+      - "5005"
+  agent:
+    image: cloudbees/jnlp-slave-with-java-build-tools:2.2.0
+    links:
+      - elk
+      - jenkins
+    # TODO there is no -noreconnect yet this does not work when first started; you need to relaunch it; work around with wait-for-it: https://docs.docker.com/compose/startup-order/
+    command: -url http://jenkins:8080/ f9bf0c290371481814f8bc235e3c53736ea9cd9f11b466b76ad794b91cb57a0b -workDir "/home/jenkins" agent

demo/external-logging-demo.iml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
+    <output url="file://$MODULE_DIR$/target/classes" />
+    <output-test url="file://$MODULE_DIR$/target/test-classes" />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src/main/groovy" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/target/generated-test-sources/injected" isTestSource="true" generated="true" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

demo/packager-config.yml

@@ -0,0 +1,99 @@
+bundle:
+  groupId: "io.jenkins.tools.war-packager.demo"
+  artifactId: "jenkins-external-task-logging-elk-demo"
+  vendor: "Jenkins project"
+  title: "Jenkins External Task Logging demo for Elasticsearch"
+  description: "Jenkins External Task Logging demo for Elasticsearch, packaged as a single WAR file. It automatically configures logging on startup"
+buildSettings:
+  docker:
+    base: "jenkins/jenkins:2.130"
+    tag: "jenkins/demo-external-task-logging-elk"
+    build: true
+war:
+  groupId: "org.jenkins-ci.main"
+  artifactId: "jenkins-war"
+  source:
+    version: 2.131-SNAPSHOT
+plugins:
+  - groupId: "org.jenkins-ci.plugins"
+    artifactId: "structs"
+    source:
+      version: 1.14
+  - groupId: "org.jenkins-ci.plugins"
+    artifactId: "scm-api"
+    source:
+      version: 2.2.7
+  - groupId: "org.jenkins-ci.plugins.workflow"
+    artifactId: "workflow-aggregator"
+    source:
+      version: 2.5
+  - groupId: "org.jenkins-ci.plugins.workflow"
+    artifactId: "workflow-api"
+    source:
+      version: 2.29-rc219.239019e84015
+    build:
+      buildOriginalVersion: true
+  - groupId: "org.jenkins-ci.plugins.workflow"
+    artifactId: "workflow-step-api"
+    source:
+      version: 2.15
+  - groupId: "org.jenkins-ci.plugins.workflow"
+    artifactId: "workflow-support"
+    source:
+      version: 2.19-rc265.3e5e4aeecfff
+    build:
+      buildOriginalVersion: true
+  - groupId: "org.jenkins-ci.plugins.workflow"
+    artifactId: "workflow-job"
+    source:
+      version: 2.22-rc311.5616213fbed0
+    build:
+      buildOriginalVersion: true
+  - groupId: "org.jenkins-ci.plugins.workflow"
+    artifactId: "workflow-durable-task-step"
+    source:
+      version: 2.20-rc333.74dc7c303e6d
+    build:
+      buildOriginalVersion: true
+  - groupId: "org.jenkins-ci.plugins"
+    artifactId: "logstash"
+    source:
+      version: 2.1.1-SNAPSHOT
+  - groupId: "io.jenkins.plugins.external-logging"
+    artifactId: "external-logging-api"
+    source:
+      version: 1.0-alpha-1-SNAPSHOT
+  - groupId: "io.jenkins.plugins.external-logging"
+    artifactId: "external-logging-elasticsearch"
+    source:
+      version: 1.0-alpha-1-SNAPSHOT
+
+  # Security warnings
+  - groupId: "org.jenkins-ci.plugins"
+    artifactId: "junit"
+    source:
+      version: 1.24
+  - groupId: "org.jenkins-ci.plugins"
+    artifactId: "mailer"
+    source:
+      version: 1.21
+  - groupId: "org.jenkins-ci.plugins"
+    artifactId: "git-client"
+    source:
+      version: 2.7.2
+  - groupId: "org.jenkins-ci.plugins"
+    artifactId: "credentials-binding"
+    source:
+      version: 1.16
+  - groupId: "org.jenkins-ci.plugins"
+    artifactId: "docker-commons"
+    source:
+      version: 1.13
+systemProperties: {
+    jenkins.model.Jenkins.slaveAgentPort: "9000",
+    jenkins.model.Jenkins.slaveAgentPortEnforce: "true"}
+groovyHooks:
+  - type: "init"
+    id: "initScripts"
+    source:
+      dir: src/main/groovy

demo/pom.xml

@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.jenkins-ci.plugins</groupId>
+    <artifactId>plugin</artifactId>
+    <version>3.15</version>
+    <relativePath />
+  </parent>
+
+  <groupId>io.jenkins.plugins.external-logging</groupId>
+  <artifactId>external-logging-demo</artifactId>
+  <name>External Logging for Elasticsearch/Logstash Demo</name>
+  <description>The plugin provides API to simplify external logging implementations for Jenkins</description>
+  <url>https://wiki.jenkins.io/display/JENKINS/External+Logging+API+Plugin</url>
+  <version>${revision}${changelist}</version>
+  <packaging>hpi</packaging>
+
+  <properties>
+    <revision>1.0-alpha-1</revision>
+    <changelist>-SNAPSHOT</changelist>
+    <jenkins.version>2.131-SNAPSHOT</jenkins.version>
+    <java.level>8</java.level>
+    <useBeta>true</useBeta>
+  </properties>
+
+  <licenses>
+    <license>
+      <name>MIT License</name>
+      <url>https://opensource.org/licenses/MIT</url>
+    </license>
+  </licenses>
+
+  <scm>
+    <connection>scm:git:git://github.com/jenkinsci/${project.artifactId}-plugin.git</connection>
+    <developerConnection>scm:git:git@github.com:jenkinsci/${project.artifactId}-plugin.git</developerConnection>
+    <url>https://github.com/jenkinsci/${project.artifactId}-plugin</url>
+    <tag>${scmTag}</tag>
+  </scm>
+
+  <dependencies>
+    <dependency>
+      <groupId>io.jenkins.plugins.external-logging</groupId>
+      <artifactId>external-logging-elasticsearch</artifactId>
+      <version>1.0-alpha-1-SNAPSHOT</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.codehaus.gmaven</groupId>
+        <artifactId>gmaven-plugin</artifactId>
+        <version>1.5-jenkins-3</version>
+        <dependencies>
+          <dependency>
+            <groupId>org.codehaus.gmaven.runtime</groupId>
+            <artifactId>gmaven-runtime-1.8</artifactId>
+            <version>1.5-jenkins-3</version>
+          </dependency>
+        </dependencies>
+        <executions>
+          <execution>
+            <goals>
+              <goal>compile</goal>
+              <goal>testCompile</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+  <repositories>
+    <repository>
+      <id>repo.jenkins-ci.org</id>
+      <url>https://repo.jenkins-ci.org/public/</url>
+    </repository>
+  </repositories>
+  <pluginRepositories>
+    <pluginRepository>
+      <id>repo.jenkins-ci.org</id>
+      <url>https://repo.jenkins-ci.org/public/</url>
+    </pluginRepository>
+  </pluginRepositories>
+
+</project>