FAB-4161 Run only patch sepecific unit test cases

Currently at CI, entire unit test cases were run which results significant long
run jobs. This change will make sure only the changed packages unit test cases
will be run at CI.

Merge patch set CI jobs will be still run the full unit test cases.
Use go test -cover as opposed to gocov for improved performance
Use git diff --name-only to find uncommitted changes
Add 'verify' target to Makefile
Add a test go source file to test the build
generate html coverage report on merge job

Also covers FAB-4192 Makefile improvements

Refine the list of PROJECT_FILES omitting things
that don't affect the build targets using that dependency
Remove unnecessary dependencies

Have check_spelling.sh and check_license.sh only work
on changed files. include last commit content for CI

Also addressed some spelling errors that slipped
in.

address binh's comment re generated files

Signed-off-by: tong li <litong01@us.ibm.com>
Signed-off-by: Christopher Ferris <chrisfer@us.ibm.com>
Signed-off-by: Rameshthoomu <rameshbabu.thoomu@gmail.com>
Change-Id: Ib2759219ced54d15fe7c9e5113bd9b1259ea68c5

Makefile

@@ -1,30 +1,22 @@
 # Copyright IBM Corp All Rights Reserved.
 # Copyright London Stock Exchange Group All Rights Reserved.
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-License-Identifier: Apache-2.0
 #
 # -------------------------------------------------------------
 # This makefile defines the following targets
 #
 #   - all (default) - builds all targets and runs all tests/checks
 #   - checks - runs all tests/checks
+#   - desk-check - runs linters and verify to test changed packages
 #   - configtxgen - builds a native configtxgen binary
 #   - cryptogen  -  builds a native cryptogen binary
 #   - peer - builds a native fabric peer binary
 #   - orderer - builds a native fabric orderer binary
 #   - release - builds release packages for the host platform
 #   - release-all - builds release packages for all target platforms
 #   - unit-test - runs the go-test based unit tests
+#   - verify - runs unit tests for only the changed package tree
 #   - test-cmd - generates a "go test" string suitable for manual customization
 #   - behave - runs the behave test
 #   - behave-deps - ensures pre-requisites are available for running behave manually
@@ -81,7 +73,11 @@ K := $(foreach exec,$(EXECUTABLES),\
 GOSHIM_DEPS = $(shell ./scripts/goListFiles.sh $(PKGNAME)/core/chaincode/shim)
 JAVASHIM_DEPS =  $(shell git ls-files core/chaincode/shim/java)
 PROTOS = $(shell git ls-files *.proto | grep -v vendor)
-PROJECT_FILES = $(shell git ls-files)
+# No sense rebuilding when non production code is changed
+PROJECT_FILES = $(shell git ls-files  | grep -v ^test | grep -v ^unit-test | \
+	grep -v ^bddtests | grep -v ^docs | grep -v _test.go$ | grep -v .md$ | \
+	grep -v ^.git | grep -v ^examples | grep -v ^devenv | grep -v .png$ | \
+	grep -v ^LICENSE )
 IMAGES = peer orderer ccenv javaenv buildenv testenv zookeeper kafka couchdb tools
 RELEASE_PLATFORMS = windows-amd64 darwin-amd64 linux-amd64 linux-ppc64le linux-s390x
 RELEASE_PKGS = configtxgen cryptogen configtxlator
@@ -98,19 +94,24 @@ include docker-env.mk
 
 all: native docker checks
 
-checks: linter license spelling unit-test behave
+checks: license spelling linter unit-test behave
+
+desk-check: license spelling linter verify behave
 
-spelling: buildenv
+.PHONY: spelling
+spelling:
 	@scripts/check_spelling.sh
 
-license: buildenv
+.PHONY: license
+license:
 	@scripts/check_license.sh
 
 .PHONY: gotools
 gotools:
 	mkdir -p build/bin
 	cd gotools && $(MAKE) install BINDIR=$(GOPATH)/bin
 
+.PHONY: gotools-clean
 gotools-clean:
 	cd gotools && $(MAKE) clean
 
@@ -145,11 +146,18 @@ testenv: build/image/testenv/$(DUMMY)
 
 couchdb: build/image/couchdb/$(DUMMY)
 
+kafka: build/image/kafka/$(DUMMY)
+
+zookeeper: build/image/zookeeper/$(DUMMY)
+
 unit-test: unit-test-clean peer-docker testenv couchdb
 	cd unit-test && docker-compose up --abort-on-container-exit --force-recreate && docker-compose down
 
 unit-tests: unit-test
 
+verify: unit-test-clean peer-docker testenv couchdb
+	cd unit-test && JOB_TYPE=VERIFY docker-compose up --abort-on-container-exit --force-recreate && docker-compose down
+
 # Generates a string to the terminal suitable for manual augmentation / re-issue, useful for running tests by hand
 test-cmd:
 	@echo "go test -ldflags \"$(GO_LDFLAGS)\""

core/endorser/endorser.go

@@ -1,17 +1,7 @@
 /*
 Copyright IBM Corp. 2016 All Rights Reserved.
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-		 http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+SPDX-License-Identifier: Apache-2.0
 */
 
 package endorser

gossip/discovery/discovery_test.go

@@ -1,17 +1,7 @@
 /*
 Copyright IBM Corp. 2016 All Rights Reserved.
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-		 http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+SPDX-License-Identifier: Apache-2.0
 */
 
 package discovery
@@ -975,7 +965,7 @@ func TestMsgStoreExpirationWithMembershipMessages(t *testing.T) {
 
 	// Checking is Alive was processed
 	for i := 0; i < peersNum; i++ {
-		checkAliveMsgExist(instances, aliveMsgs, i, "[Step 1 - proccessing aliveMsg]")
+		checkAliveMsgExist(instances, aliveMsgs, i, "[Step 1 - processing aliveMsg]")
 	}
 
 	// Creating MembershipResponse while all instances have full membership

gotools/Makefile

@@ -18,7 +18,7 @@ OBJDIR ?= build
 TMP_GOPATH=$(OBJDIR)/gopath
 GOBIN=$(abspath $(TMP_GOPATH)/bin)
 
-GOTOOLS = golint govendor goimports protoc-gen-go ginkgo gocov gocov-xml misspell
+GOTOOLS = golint govendor goimports protoc-gen-go ginkgo gocov gocov-html misspell
 GOTOOLS_BIN = $(patsubst %,$(GOBIN)/%, $(GOTOOLS))
 
 # go tool->path mapping
@@ -27,9 +27,8 @@ go.fqp.golint    := github.com/golang/lint/golint
 go.fqp.goimports := golang.org/x/tools/cmd/goimports
 go.fqp.ginkgo    := github.com/onsi/ginkgo/ginkgo
 go.fqp.gocov     := github.com/axw/gocov/...
-go.fqp.gocov-xml := github.com/AlekSi/gocov-xml
 go.fqp.misspell  := github.com/client9/misspell/cmd/misspell
-
+go.fqp.gocov-html:= github.com/matm/gocov-html
 all: $(GOTOOLS_BIN)
 
 install: $(GOTOOLS_BIN)

protos/utils/txutils_test.go

@@ -1,17 +1,7 @@
 /*
 Copyright IBM Corp. 2017 All Rights Reserved.
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-		 http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+SPDX-License-Identifier: Apache-2.0
 */
 
 package utils_test
@@ -218,7 +208,7 @@ func TestCreateSignedTx(t *testing.T) {
 
 	//
 	//
-	// addtional failure cases
+	// additional failure cases
 	prop = &pb.Proposal{}
 	responses = []*pb.ProposalResponse{}
 	// no proposal responses

scripts/check_license.sh

@@ -5,13 +5,24 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+CHECK=$(git diff --name-only HEAD * | grep -v .png$ | grep -v .rst$ | grep -v .git \
+  | grep -v .md$ | grep -v ^vendor/ | grep -v ^build/ | grep -v .pb.go$ | sort -u)
 
-echo "Checking Go files for license headers ..."
-missing=`find . -name "*.go" | grep -v build/ | grep -v vendor/ | grep -v ".pb.go" | grep -v "examples/chaincode/go/utxo/consensus/consensus.go" | xargs grep -l -L "Apache License"`
+if [[ -z "$CHECK" ]]; then
+  CHECK=$(git diff-tree --no-commit-id --name-only -r $(git log -2 \
+    --pretty=format:"%h") | grep -v .png$ | grep -v .rst$ | grep -v .git \
+    | grep -v .md$ | grep -v ^vendor/ | grep -v ^build/ | grep -v .pb.go$ | sort -u)
+fi
+
+echo "Checking committed files for SPDX-License-Identifier headers ..."
+missing=`echo $CHECK | xargs grep -L "SPDX-License-Identifier"`
 if [ -z "$missing" ]; then
-   echo "All go files have license headers"
+   echo "All files have SPDX-License-Identifier headers"
    exit 0
 fi
-echo "The following files are missing license headers:"
+echo "The following files are missing SPDX-License-Identifier headers:"
 echo "$missing"
+echo
+echo "Please replace the Apache license header comment text with:"
+echo "SPDX-License-Identifier: Apache-2.0"
 exit 1

scripts/check_spelling.sh

@@ -5,9 +5,17 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+CHECK=$(git diff --name-only HEAD * | grep -v .png$ | grep -v .git \
+  | grep -v ^vendor/ | grep -v ^build/ | sort -u)
 
-echo "Checking Go files for spelling errors ..."
-errs=`find . -name "*.go" | grep -v vendor/ | grep -v build/ | grep -v ".pb.go" | xargs misspell`
+if [[ -z "$CHECK" ]]; then
+  CHECK=$(git diff-tree --no-commit-id --name-only -r $(git log -2 \
+    --pretty=format:"%h") | grep -v .png$ | grep -v .git \
+    | grep -v ^vendor/ | grep -v ^build/ | sort -u)
+fi
+
+echo "Checking changed go files for spelling errors ..."
+errs=`echo $CHECK | xargs misspell -source=text`
 if [ -z "$errs" ]; then
    echo "spell checker passed"
    exit 0

test/chaincodes/AuctionApp/art.go

@@ -1,19 +1,9 @@
-/******************************************************************
+/*
 Copyright IT People Corp. 2017 All Rights Reserved.
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+SPDX-License-Identifier: Apache-2.0
 
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-******************************************************************/
+*/
 
 ///////////////////////////////////////////////////////////////////////
 // Author : IT People - Mohan Venkataraman - Auction API for v1.0
@@ -394,7 +384,7 @@ func (t *SimpleChaincode) query(stub shim.ChaincodeStubInterface, function strin
 
 	if len(args) < 1 {
 		fmt.Println("Query() : Include at least 1 arguments Key ")
-		return shim.Error("Query() : Expecting Transation type and Key value for query")
+		return shim.Error("Query() : Expecting Transaction type and Key value for query")
 	}
 
 	QueryRequest := QueryFunction(function)
@@ -457,7 +447,7 @@ func GetUser(stub shim.ChaincodeStubInterface, function string, args []string) p
 		return shim.Error(jsonResp)
 	}
 
-	fmt.Println("GetUser() : Response : Successfull -")
+	fmt.Println("GetUser() : Response : Successful -")
 	return shim.Success(Avalbytes)
 }
 
@@ -491,7 +481,7 @@ func GetItem(stub shim.ChaincodeStubInterface, function string, args []string) p
 	itemObj.ItemImage = []byte{}
 	Avalbytes, _ = ARtoJSON(itemObj)
 
-	fmt.Println("GetItem() : Response : Successfull ")
+	fmt.Println("GetItem() : Response : Successful ")
 	return shim.Success(Avalbytes)
 }
 
@@ -546,7 +536,7 @@ func ValidateItemOwnership(stub shim.ChaincodeStubInterface, function string, ar
 		return shim.Error(jsonResp)
 	}
 
-	fmt.Print("ValidateItemOwnership() : Response : Successfull - \n")
+	fmt.Print("ValidateItemOwnership() : Response : Successful - \n")
 	return shim.Success(Avalbytes)
 }
 
@@ -575,7 +565,7 @@ func GetAuctionRequest(stub shim.ChaincodeStubInterface, function string, args [
 		return shim.Error(jsonResp)
 	}
 
-	fmt.Println("GetAuctionRequest() : Response : Successfull - \n")
+	fmt.Println("GetAuctionRequest() : Response : Successful - \n")
 	return shim.Success(Avalbytes)
 }
 
@@ -611,7 +601,7 @@ func GetBid(stub shim.ChaincodeStubInterface, function string, args []string) pb
 		return shim.Error(jsonResp)
 	}
 
-	fmt.Println("GetBid() : Response : Successfull -")
+	fmt.Println("GetBid() : Response : Successful -")
 	return shim.Success(Avalbytes)
 }
 
@@ -639,7 +629,7 @@ func GetTransaction(stub shim.ChaincodeStubInterface, function string, args []st
 		return shim.Error(jsonResp)
 	}
 
-	fmt.Println("GetTransaction() : Response : Successfull")
+	fmt.Println("GetTransaction() : Response : Successful")
 	return shim.Success(Avalbytes)
 }
 
@@ -649,7 +639,7 @@ func GetTransaction(stub shim.ChaincodeStubInterface, function string, args []st
 // There are different types of users - Traders (TRD), Auction Houses (AH)
 // Shippers (SHP), Insurance Companies (INS), Banks (BNK)
 // While this version of the chain code does not enforce strict validation
-// the business process recomends validating each persona for the service
+// the business process recommends validating each persona for the service
 // they provide or their participation on the auction blockchain, future enhancements will do that
 // ./peer chaincode invoke -l golang -n mycc -c '{"Function": "PostUser", "Args":["100", "USER", "Ashley Hart", "TRD",  "Morrisville Parkway, #216, Morrisville, NC 27560", "9198063535", "ashley@itpeople.com", "SUNTRUST", "00017102345", "0234678"]}'
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -934,7 +924,7 @@ func TransferItem(stub shim.ChaincodeStubInterface, function string, args []stri
 	myItem.AES_Key, _ = GenAESKey()
 	myItem.ItemImage = Encrypt(myItem.AES_Key, image)
 
-	// Update the owner to the new owner transfered to
+	// Update the owner to the new owner transferred to
 	myItem.CurrentOwnerID = args[3]
 
 	ar, err = ARtoJSON(myItem)
@@ -1193,7 +1183,7 @@ func PostTransaction(stub shim.ChaincodeStubInterface, function string, args []s
 
 	buyer := response.Payload
 
-	fmt.Println("PostTransaction(): Validated Buyer information successfully ", buyer, ar.UserId)
+	fmt.Println("PostTransaction(): Validated Buyer information Successfully ", buyer, ar.UserId)
 
 	// Validate Item record
 	response = ValidateItemSubmission(stub, ar.ItemID)
@@ -1203,7 +1193,7 @@ func PostTransaction(stub shim.ChaincodeStubInterface, function string, args []s
 	}
 
 	lastUpdatedItemOBCObject := response.Payload
-	fmt.Println("PostTransaction() : Validated Item Object in Blockchain successfully", ar.ItemID)
+	fmt.Println("PostTransaction() : Validated Item Object in Blockchain Successfully", ar.ItemID)
 
 	// Update Item Object with new Owner Key
 	response = UpdateItemObject(stub, lastUpdatedItemOBCObject, ar.HammerPrice, ar.UserId)
@@ -1216,7 +1206,7 @@ func PostTransaction(stub shim.ChaincodeStubInterface, function string, args []s
 		fmt.Println("PostTransaction() : New encryption Key is  ", newKey)
 	}
 
-	fmt.Println("PostTransaction() : Updated Item Master Object in Blockchain successfully", ar.ItemID)
+	fmt.Println("PostTransaction() : Updated Item Master Object in Blockchain Successfully", ar.ItemID)
 
 	// Post an Item Log
 	itemObject, err := JSONtoAR(lastUpdatedItemOBCObject)
@@ -1235,7 +1225,7 @@ func PostTransaction(stub shim.ChaincodeStubInterface, function string, args []s
 		return shim.Error(err.Error())
 	}
 
-	fmt.Println("PostTransaction() : Inserted item log record successfully", ar.ItemID)
+	fmt.Println("PostTransaction() : Inserted item log record Successfully", ar.ItemID)
 
 	// Convert Transaction Object to JSON
 	buff, err := TrantoJSON(ar) //
@@ -1252,7 +1242,7 @@ func PostTransaction(stub shim.ChaincodeStubInterface, function string, args []s
 		return shim.Error(err.Error())
 	}
 
-	fmt.Println("PostTransaction() : Posted Transaction Record successfully\n")
+	fmt.Println("PostTransaction() : Posted Transaction Record Successfully\n")
 
 	// Returns New Key. To get Transaction Details, run GetTransaction
 
@@ -2545,7 +2535,7 @@ func ProcessQueryResult(stub shim.ChaincodeStubInterface, Avalbytes []byte, args
 			fmt.Println("ProcessRequestType() : Image decrytion failed ")
 			return err
 		}
-		fmt.Println("ProcessRequestType() : Image conversion from byte[] to file successfull ")
+		fmt.Println("ProcessRequestType() : Image conversion from byte[] to file Successful ")
 		err = ByteArrayToImage(image, "copy."+ar.ItemPicFN)
 		if err != nil {