FABC-833 Update Jenkinsfile

This patch fixes the minor issues observed in Jenkinsfile
and update the CI doc.

Change-Id: I859844a2c17398675fa0a86c49a76793863975fb
Signed-off-by: rameshthoomu <rameshbabu.thoomu@gmail.com>

Jenkinsfile

@@ -26,6 +26,8 @@ timestamps { // set the timestamps on the jenkins console
 
 def buildStages() {
   try {
+    // LF team has to install the newer version in Jenkins global config
+    // Send an email to helpdesk@hyperledger.org to add newer version
     def nodeHome = tool 'nodejs-8.14.0'
     def ROOTDIR = pwd()
     stage('Clean Environment') {
@@ -54,18 +56,18 @@ def buildStages() {
       env.PATH = "$GOROOT/bin:$GOPATH/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:${nodeHome}/bin:$PATH"
     }
 
-      if (DOC_CHANGE > 0 && CODE_CHANGE == 0) {
-        println "ONLY DOC BUILD"
+      if (DOC_CHANGE > '0' && CODE_CHANGE == '0') {
+        sh "echo -e \033[1m ONLY DOC BUILD\033[0m"
         docsBuild()
-      } else if (DOC_CHANGE > 0 && CODE_CHANGE > 0) {
-          println "CODE AND DOC BUILD"
+      } else if (DOC_CHANGE > '0' && CODE_CHANGE > '0') {
+          sh "echo -e \033[1m CODE AND DOC BUILD\033[0m"
           basicChecks()
           docsBuild()
           runTests()
       } else {
-          println "CODE BUILD"
+          sh "echo -e \033[1m CODE BUILD\033[0m"
           basicChecks() // basic checks
-          println "CODE TESTS"
+          sh "echo -e \033[1m CODE TESTS\033[0m"
           runTests() // e2e on merge and unit, fvt tests on parallel
       }
     } finally { // post build actions
@@ -103,7 +105,6 @@ def buildStages() {
 def docsBuild () {
   def ROOTDIR = pwd()
   stage("Docs Build") {
-	  wrap([$class: 'AnsiColorBuildWrapper', 'colorMapName': 'xterm']) {
       try {
         dir("$ROOTDIR/$BASE_DIR") {
           sh ''' set +x -ue
@@ -119,7 +120,6 @@ def docsBuild () {
           currentBuild.result = 'FAILURE'
           throw err
       }
-    }
   }
 }
 
@@ -155,8 +155,7 @@ def fabCar() {
           fabBuildLibrary.deleteContainers()
           // Delete unused docker images (none,dev,test-vp etc..)
           fabBuildLibrary.deleteUnusedImages()
-          sh 'docker ps -a && docker images'
-          dir("$ROOTDIR/gopath/src/github.com/hyperledger/fabric-samples/scripts/Jenkins_Scripts") {
+          dir("$ROOTDIR/gopath/src/github.com/hyperledger/fabric-samples/scripts/ci_scripts") {
             sh './fabcar.sh'
           }
         } catch (err) {
@@ -182,7 +181,6 @@ def e2e_sdk_node() {
         fabBuildLibrary.deleteContainers()
         // Delete unused docker images (none,dev,test-vp etc..)
         fabBuildLibrary.deleteUnusedImages()
-        sh 'docker ps -a && docker images'
         dir("$ROOTDIR/gopath/src/github.com/hyperledger/fabric-sdk-node") {
           sh '''set +x -ue
             npm install
@@ -322,7 +320,7 @@ def runTests() {
       },
       failFast: true ) // Stop the build flow if one job fails
     stage("Unstash") {
-      if (DOC_CHANGE > 0 && CODE_CHANGE == 0) {
+      if (DOC_CHANGE > '0' && CODE_CHANGE == '0') {
         // unstash not required for doc only builds
         println "Unstash not required"
       } else {
@@ -341,4 +339,4 @@ def runTests() {
         }
       }
     } // stage parallel
-} // runTests
+} // runTests

README.md

@@ -153,6 +153,7 @@ Following are the steps to update cfssl package using version 1.0.8 of govendor
 
 Please have a look at [Continuous Integration Process](docs/source/ca-ci.md)
 
+
 ## License <a name="license"></a>
 
 Hyperledger Project source code files are made available under the Apache License, Version 2.0 (Apache-2.0), located in the [LICENSE](LICENSE) file. Hyperledger Project documentation files are made available under the Creative Commons Attribution 4.0 International License (CC-BY-4.0), available at http://creativecommons.org/licenses/by/4.0/.

ci.properties

@@ -1,12 +1,12 @@
-# Set base version
+# Set base version from fabric branch
 FAB_BASE_VERSION=2.0.0
-# Set base image version
-FAB_BASEIMAGE_VERSION=0.4.14
+# Set base image version from fabric branch
+FAB_BASEIMAGE_VERSION=0.4.15
 # Pull below list of images from Hyperledger Docker Hub
 FAB_THIRDPARTY_IMAGES_LIST=kafka zookeeper couchdb
-# Set related rocketChat channel name. Default: jenkins-robot
-CHANNEL_NAME=jenkins-robot
 # Set compaitable go version
 GO_VER=1.11.5
-# Pull below list of images from nexus3
+# Pull below list of images from nexus3 for e2e tests
 FAB_IMAGES_LIST=javaenv nodeenv
+# Set related rocketChat channel name. Default: jenkins-robot
+CHANNEL_NAME=jenkins-robot

docs/source/ca-ci.md

@@ -1,22 +1,33 @@
 # Continuous Integration Process
 
-This document explains the fabric-ca Jenkins pipeline flow and FAQ's on the build process to help developer to get more familiarize with the process flow.
+This document explains the fabric-ca Jenkins pipeline flow and FAQ's on the build process
+to help developer to get more familiarize with the process flow.
 
-We use Jenkins as a CI tool and to manage jobs, we use [JJB](https://docs.openstack.org/infra/jenkins-job-builder). Please see the pipeline job configuration template here https://ci-docs.readthedocs.io/en/latest/source/pipeline_jobs.html#job-templates.
+We use Jenkins as a CI tool and to manage jobs, we use [JJB](https://docs.openstack.org/infra/jenkins-job-builder).
+Please see the pipeline job configuration template here https://ci-docs.readthedocs.io/en/latest/source/pipeline_jobs.html#job-templates.
 
 ## CI Pipeline flow
 
-- Every Gerrit patchset triggers a verify job with the Gerrit Refspec on the parent commit of the patchset and run the below tests from the `Jenkinsfile`. Note: When you are ready to merge a patchset, it's always a best practice to rebase the patchset on the latest commit.
+- Every Gerrit patchset triggers a verify job with the Gerrit Refspec on the parent commit
+  of the patchset and run the below tests from the `Jenkinsfile`. Note: When you are ready
+  to merge a patchset, it's always a best practice to rebase the patchset on the latest commit.
 
     - Basic Checks (make checks)
     - Documentation build (tox -edocs)
     - Unit tests (make unit-tests)
     - FVT tests (make fvt-tests)
     - E2E tests (Only after patchset is merged)
 
-All the above tests run on the Hyperledger infarstructure x86_64 build nodes. All these nodes uses the packer with pre-configured software packages. This helps us to run the tests in much faster than installing required packages everytime.
+All the above tests run on the Hyperledger infarstructure x86_64 build nodes. All these nodes
+uses the packer with pre-configured software packages. This helps us to run the tests in much
+faster than installing required packages everytime.
 
-Below steps shows what each stage does in the Jenkins pipeline verify and merge flow. Every Gerrit patchset triggers the fabric-ca-verify-x86_64 job and runs the below tests on x86_64 platform. Before execute the below tests, it clean the environment (Deletes the left over build artifiacts) and clone the repository with the Gerrit Refspec.
+Below steps shows what each stage does in the Jenkins pipeline verify and merge flow.
+Every Gerrit patchset triggers the fabric-ca-verify-x86_64 job and runs the below tests on
+x86_64 platform. Before execute the below tests, it clean the environment (Deletes the left
+over build artifiacts) and clone the repository with the Gerrit Refspec. Based on the file
+extenstions, Jenkins pipeline script triggers the stages. If the patchset contains specific
+doc extension, it only triggers **Docs Build** Stage otherwise it triggers all stages.
 
 ![](images/pipeline_flow.png)
 
@@ -29,77 +40,106 @@ Below steps shows what each stage does in the Jenkins pipeline verify and merge
 
 #### Docs Build
 
-- This stage gets triggered only when a patchset contains .md, .rst etc doc related file exetensions.
 - We run `tox -edocs` from the root directory.
-- Displays the output in the form of HTML Publisher on the `fabric-ca-verify-x86_64` job. Click on **Docs Output** link on the Jenkins console.
+- Displays the output in the form of HTML Publisher on the `fabric-ca-verify-x86_64` job.
+  Click on **Docs Output** link on the Jenkins console.
 
 #### Unit Tests
 
-- We run `make unit-test` target to run the go based unit-tests and publish the coverage report on the Jenkins console. Click on **Coverage Report** link on the Jenkins console.
+- We run `make unit-test` target to run the go based unit-tests and publish the coverage
+  report on the Jenkins console. Click on **Coverage Report** link on the Jenkins console
+  to see the code coverage.
 
 #### FVT Tests
 
-- We run `make fvt-tests` target to fun fvt tests, which includes tests that perform end-to-end test scenarios with PosgreSQL and MySQL databases. These tests include database migration, backwards compatibility, and LDAP integration. https://github.com/hyperledger/fabric-ca/blob/master/scripts/fvt/README.md
+- We run `make fvt-tests` target to run fvt tests, which includes tests that performs end-to-end
+  test scenarios with PosgreSQL and MySQL databases. These tests include database migration,
+  backwards compatibility, and LDAP integration. https://github.com/hyperledger/fabric-ca/blob/master/scripts/fvt/README.md
 
 #### E2E tests
 
-- We run **e2e tests** in the **merge job** and it performs the following tests. The intention of running e2e tests as part of the merge job is to test the dependent tests of fabric-ca.
-        - fabcar
-        - fabric-sdk-node - We run **gulp run-end-to-end** target which executes most of the end to end tests of fabric-sdk-node which are depend on fabric-ca.
-        - fabric-sdk-java - We run **ci_run.sh** script which is pre-baked in fabric-sdk-java repository.
+- We run **e2e tests** in the **merge job** and it performs the following tests. The intention
+  of running e2e tests as part of the merge job is to test the dependent tests of fabric-ca.
+    - fabcar
+    - fabric-sdk-node - We run **gulp run-end-to-end** target which executes most of the end to end
+      tests of fabric-sdk-node which are depend on fabric-ca.
+    - fabric-sdk-java - We run **ci_run.sh** script which is pre-baked in fabric-sdk-java repository.
 
-As we trigger `fabric-ca-verify-x86_64` and `fabric-ca-merge-x86_64` pipeline jobs for every gerrit patchset, we execute the tests in the below order.
+As we trigger `fabric-ca-verify-x86_64` and `fabric-ca-merge-x86_64` pipeline jobs for every gerrit patchset,
+we execute these tests in the below order.
 
-After the DocsBuild is passed, Jenkins Pipeline triggers the Unit and FVT Tests parallel on two different nodes. After the tests are executed successfully it posts a Gerrit voting on the patchset. If DocsBuild fails, it send the result back to Gerrit patchset and it won't trigger further builds.
+After the DocsBuild stage is passed, Jenkins Pipeline triggers the Unit and FVT Tests parallel
+on two different nodes. After the tests are executed successfully it posts a Gerrit voting
+on the patchset. If DocsBuild stage fails, it send the result back to Gerrit patchset and it
+won't trigger further builds.
 
 See below **FAQ's** for more information on the pipeline process.
 
 ## FAQ's
 
 #### What happens on the merge job?
 
-After the patchset got merged in the fabric-ca repository, it follows the above pipeline flow and executes the e2e tests in parallel to the Unit and FVT Tests.
+After the patchset got merged in the fabric-ca repository, it follows the above pipeline flow and
+executes the e2e tests in parallel to the Unit and FVT Tests.
 
 **Merge Pipeline Flow**
 
 ```
 CleanEnvironment -- OutputEnvironment -- CloneRefSpec -- BasicChecks -- DocsBuild - Tests (E2E, Unit, FVT Tests)
 ```
 
-Jenkins clones the latest merged commit and runs the below steps
+Jenkins clones the latest merged commit and executes the below steps
 
-- Build fabric, fabric-ca images & Binaries
-- Pull Thirdparty Images (Couchdb, zookeeper, kafka)
+- Build fabric, fabric-ca images & binaries
+- Pull Thirdparty Images from DockerHub (Couchdb, zookeeper, kafka)
 - Pull javaenv, nodeenv images from nexus3 (latest stable images published after successful merge job of each repo)
+- Tests
   - fabcar tests
   - fabric-sdk-node (npm install, gulp run-end-to-end)
   - fabric-sdk-java (Run ci_run.sh)
 
 #### What happens if one of the build stage fails?
 
-As we are running these tests in `fastFailure: true` (if any build stage fails in the parallel process, it will terminate/abort the current running tests and sends the result back to the Gerrit Patchset. This way, CI will avoid runnning tests when there is a failure in one of the parallel build stage.
+As we are running these tests in `fastFailure: true` (if any build stage fails in the parallel
+process, it will terminate/abort the current running tests and sends the result back to the
+Gerrit Patchset. This way, CI will avoid runnning tests when there is a failure in one of the
+parallel build stage.
 
 It shows `aborted` on the aborted stage on pipeline staged view.
 
 #### How to re-trigger failed tests?
 
-With this pipeline flow, you can not re-trigger specific failed job, instead you can post comments `reverify` or `reverify-x` on the gerrit patchset to trigger the `fabric-ca-verify-x86_64` job which triggers the pipeline flow as mentioned above. Also, we provided `remerge` or `remerge-x` comment phrases to re-trigger the failed merge jobs.
+With this pipeline flow, you can NOT re-trigger the specific stage, but you can post comments
+`reverify` or `reverify-x` on the gerrit patchset to trigger the `fabric-ca-verify-x86_64`
+job which triggers the pipeline flow as mentioned above. Also, we provided `remerge` or `remerge-x`
+comment phrases to re-trigger the failed merge jobs.
 
 #### Where should I see the output of the stages?
 
-Piepline supports two views (stages and blueocean). Staged views shows on the Jenkins job main page and it shows each stage in order and the status. For better view, we suggest you to access BlueOcean plugin. Click on the JOB Number and click on the **Open Blue Ocean** link that shows the build stages in pipeline view.
+Piepline supports two views (staged and blueocean). **Staged views** shows on the Jenkins job main
+page and it shows each stage in order and the status. For better view, we suggest you to access
+BlueOcean plugin. Click on the JOB Number and click on the **Open Blue Ocean** link that shows
+the build stages in pipeline view.
 
 #### How to add more stages to this pipeline flow?
 
-We use scripted pipeline syntax with groovy and shell scripts. Also, we use global shared library scripts which are placed in https://github.com/hyperledger/ci-management/tree/master/vars. Try to leverage these common functions in your code. All you have to do is, undestand the pipeline flow of the tests and conditions, add more stages as mentioned in the existing Jenkinsfile.
+We use scripted pipeline syntax with groovy and shell scripts. Also, we use global shared library
+scripts which are placed in https://github.com/hyperledger/ci-management/tree/master/vars.
+Try to leverage these common functions in your code. All you have to do is, undestand the pipeline
+flow of the tests and conditions, add more stages as mentioned in the existing Jenkinsfile.
 
 #### How will I get build failure notifications?
 
-On every merge failure, we send build failure email notications to the submitter of the patchset and send build details to the Rocket Chat **jenkins-robot** channel. Check the result here https://chat.hyperledger.org/channel/jenkins-robot
+On every merge failure, we send build failure email notications to the submitter of the patchset
+and send build details to the Rocket Chat **jenkins-robot** channel. Check the result here
+https://chat.hyperledger.org/channel/jenkins-robot
 
 #### What steps I have to modify when I create a new branch from master?
 
-As the Jenkinsfile is completely parametrzed, you no need to modify anything in the Jenkinsfile but you may endup modifying **ci.properties** file with the appropirate Base Versions, Baseimage versions etc... in the new branch. We suggest you to modify this file immediately after you create a new branch to avoid running tests on old versions.
+As the Jenkinsfile is completely parametrzed, you no need to modify anything in the Jenkinsfile
+but you may endup modifying **ci.properties** file with the appropirate Base Versions,
+Baseimage versions etc... in the new branch. We suggest you to modify this file immediately
+after you create a new branch to avoid running tests on old versions.
 
 #### What are the supported platforms
 
@@ -112,14 +152,18 @@ We use global shared library scripts and Jenkinsfile along with the build file.
 
 Global Shared Library - https://github.com/hyperledger/ci-management/tree/master/vars
 
-Jenkinsfile           - https://github.com/hyperledger/fabric-sdk-node/tree/master/Jenkinsfile
+Jenkinsfile           - https://github.com/hyperledger/fabric-ca/tree/master/Jenkinsfile
 
 ci.properties         - https://github.com/hyperledger/fabric-ca/tree/master/ci.properties
 (ci.properties is the only file you have to modify with the values requried for the specific branch.)
 
 Packer Scripts        - https://github.com/hyperledger/ci-management/blob/master/packer/provision/docker.sh
-(Packer is a tool for automatically creating VM and container images, configuring them and post-processing them into standard output formats. We build Hyperledger's CI images via Packer and attach them to x86_64 build nodes. On s390x, we install manually. See the packages we install as a pre-requisite in the CI x86 build nodes.)
+(Packer is a tool for automatically creating VM and container images, configuring them and
+post-processing them into standard output formats. We build Hyperledger's CI images via Packer
+and attach them to x86_64 build nodes. On s390x, we install manually. See the packages we
+install as a pre-requisite in the CI x86 build nodes.)
 
 #### How to reach out to CI team?
 
 Post your questions or feedback in https://chat.hyperledger.org/channel/ci-pipeline or https://chat.hyperledger.org/channel/fabric-ci Rocket Chat channels.
+You can also create a JIRA task or bug in FABCI project. https://jira.hyperledger.org/projects/FABCI