Use batch git hasher everywhere with different read modes

cmd/acceptance_test.go

@@ -298,7 +298,7 @@ func TestIgnoreHistoryDetectsExistingIssuesOnHead(t *testing.T) {
 }
 
 func runTalismanInPrePushMode(git *git_testing.GitTesting) int {
-	options.Debug = false
+	options.Debug = true
 	options.GitHook = PrePush
 	return runTalisman(git)
 }

cmd/checksum_cmd.go

@@ -6,6 +6,8 @@ import (
 	"talisman/checksumcalculator"
 	"talisman/gitrepo"
 	"talisman/utility"
+
+	"github.com/sirupsen/logrus"
 )
 
 type ChecksumCmd struct {
@@ -21,8 +23,19 @@ func (s *ChecksumCmd) Run() int {
 	repo := gitrepo.RepoLocatedAt(wd)
 	gitTrackedFilesAsAdditions := repo.TrackedFilesAsAdditions()
 	gitTrackedFilesAsAdditions = append(gitTrackedFilesAsAdditions, repo.StagedAdditions()...)
-	cc := checksumcalculator.NewChecksumCalculator(utility.MakeHasher("checksum", wd), gitTrackedFilesAsAdditions)
+	hasher := utility.MakeHasher("checksum", wd)
+
+	err := hasher.Start()
+	if err != nil {
+		logrus.Errorf("unable to start hasher: %v", err)
+		return EXIT_FAILURE
+	}
+
+	cc := checksumcalculator.NewChecksumCalculator(hasher, gitTrackedFilesAsAdditions)
+	hasher.Shutdown()
+
 	rcSuggestion := cc.SuggestTalismanRC(s.fileNamePatterns)
+
 	if rcSuggestion != "" {
 		fmt.Print(rcSuggestion)
 		return EXIT_SUCCESS

cmd/scanner_cmd.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"talisman/detector"
 	"talisman/detector/helpers"
 	"talisman/gitrepo"
@@ -27,7 +28,7 @@ type ScannerCmd struct {
 func (s *ScannerCmd) Run(tRC *talismanrc.TalismanRC) int {
 	fmt.Printf("\n\n")
 	utility.CreateArt("Running ScanMode..")
-	detector.DefaultChain(tRC, SCAN_MODE).Test(s.additions, tRC, s.results)
+	detector.DefaultChain(tRC, "default").Test(s.additions, tRC, s.results)
 	reportsPath, err := report.GenerateReport(s.results, s.reportDirectory)
 	if err != nil {
 		logr.Errorf("error while generating report: %v", err)
@@ -47,7 +48,9 @@ func (s *ScannerCmd) exitStatus() int {
 
 //NewScannerCmd Returns a new scanner command
 func NewScannerCmd(ignoreHistory bool, reportDirectory string) *ScannerCmd {
-	additions := scanner.GetAdditions(ignoreHistory)
+	repoRoot, _ := os.Getwd()
+	reader := gitrepo.NewBatchGitObjectHashReader(repoRoot)
+	additions := scanner.GetAdditions(ignoreHistory, reader)
 	return &ScannerCmd{
 		additions:       additions,
 		results:         helpers.NewDetectionResults(talismanrc.ScanMode),

detector/chain.go

@@ -51,6 +51,7 @@ func (dc *Chain) Test(currentAdditions []gitrepo.Addition, talismanRC *talismanr
 	repo := gitrepo.RepoLocatedAt(wd)
 	allAdditions := repo.TrackedFilesAsAdditions()
 	hasher := utility.MakeHasher(dc.mode, wd)
+	hasher.Start()
 	calculator := checksumcalculator.NewChecksumCalculator(hasher, append(allAdditions, currentAdditions...))
 	cc := helpers.NewChecksumCompare(calculator, hasher, talismanRC)
 	log.Printf("Number of files to scan: %d\n", len(currentAdditions))
@@ -64,4 +65,5 @@ func (dc *Chain) Test(currentAdditions []gitrepo.Addition, talismanRC *talismanr
 		})
 	}
 	progressBar.Finish()
+	hasher.Shutdown()
 }

gitrepo/git_readers.go

@@ -10,13 +10,34 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-type Reader func(string) ([]byte, error)
+type ReadFunc func(string) ([]byte, error)
+type BatchReader interface {
+	Start() error
+	Read(string) ([]byte, error)
+	Shutdown() error
+}
+
+type BatchGitObjectReader struct {
+	repo         *GitRepo
+	cmd          *exec.Cmd
+	inputWriter  *bufio.Writer
+	outputReader *bufio.Reader
+	read         ReadFunc
+}
 
-func NewCommittedRepoFileReader(wd string) Reader {
-	return makeReader(wd, GIT_HEAD_PREFIX)
+func (bgor *BatchGitObjectReader) Start() error {
+	return bgor.cmd.Start()
+}
+
+func (bgor *BatchGitObjectReader) Shutdown() error {
+	return bgor.cmd.Process.Kill()
+}
+
+func (bgor *BatchGitObjectReader) Read(expr string) ([]byte, error) {
+	return bgor.read(expr)
 }
 
-func NewBatchGitObjectReader(root string) Reader {
+func newBatchGitObjectReader(root string) *BatchGitObjectReader {
 	repo := &GitRepo{root}
 	cmd := repo.makeRepoCommand("git", "cat-file", "--batch=%(objectsize)")
 	inputPipe, err := cmd.StdinPipe()
@@ -33,55 +54,68 @@ func NewBatchGitObjectReader(root string) Reader {
 		inputWriter:  bufio.NewWriter(inputPipe),
 		outputReader: bufio.NewReader(outputPipe),
 	}
-	err = batchReader.start()
-	if err != nil {
-		logrus.Fatalf("error starting batch command: %v", err)
-	}
-	return batchReader.makeReader()
+	return &batchReader
 }
 
-func NewRepoFileReader(wd string) Reader {
-	return makeReader(wd, GIT_STAGED_PREFIX)
+func NewBatchGitHeadPathReader(root string) BatchReader {
+	bgor := newBatchGitObjectReader(root)
+	bgor.read = bgor.makePathReader(GIT_HEAD_PREFIX)
+	return bgor
 }
 
-func makeReader(repoRoot, prefix string) Reader {
-	return func(path string) ([]byte, error) {
-		return GitRepo{repoRoot}.readRepoFile(path, prefix)
-	}
+type batchGitHeadPathReader struct {
+	bgor *BatchGitObjectReader
 }
 
-type BatchGitObjectReader struct {
-	repo         *GitRepo
-	cmd          *exec.Cmd
-	inputWriter  *bufio.Writer
-	outputReader *bufio.Reader
+func NewBatchGitStagedPathReader(root string) BatchReader {
+	bgor := newBatchGitObjectReader(root)
+	bgor.read = bgor.makePathReader(GIT_STAGED_PREFIX)
+	return bgor
 }
 
-func (bgor *BatchGitObjectReader) start() error {
-	return bgor.cmd.Start()
+type batchGitStagedPathReader struct {
+	bgor *BatchGitObjectReader
+}
+
+func NewBatchGitObjectHashReader(root string) BatchReader {
+	bgor := newBatchGitObjectReader(root)
+	bgor.read = bgor.makeObjectHashReader()
+	return bgor
 }
 
 type gitCatFileReadResult struct {
 	contents []byte
 	err      error
 }
 
-func (bgor *BatchGitObjectReader) makeReader() Reader {
-	pathChan := make(chan (string))
+func (bgor *BatchGitObjectReader) makePathReader(prefix string) ReadFunc {
+	pathChan := make(chan ([]byte))
 	resultsChan := make(chan (gitCatFileReadResult))
 	go bgor.doCatFile(pathChan, resultsChan)
 	return func(path string) ([]byte, error) {
-		pathChan <- path
+		pathChan <- []byte(prefix + ":" + path)
+		result := <-resultsChan
+		return result.contents, result.err
+	}
+}
+
+func (bgor *BatchGitObjectReader) makeObjectHashReader() ReadFunc {
+	objectHashChan := make(chan ([]byte))
+	resultsChan := make(chan (gitCatFileReadResult))
+	go bgor.doCatFile(objectHashChan, resultsChan)
+	return func(objectHash string) ([]byte, error) {
+		objectHashChan <- []byte(objectHash)
 		result := <-resultsChan
 		return result.contents, result.err
 	}
 }
 
-func (bgor *BatchGitObjectReader) doCatFile(pathChan chan (string), resultsChan chan (gitCatFileReadResult)) {
+func (bgor *BatchGitObjectReader) doCatFile(gitExpressionChan chan ([]byte), resultsChan chan (gitCatFileReadResult)) {
 	for {
-		path := <-pathChan
+		gitExpression := <-gitExpressionChan
+		gitExpression = append(gitExpression, '\n')
 		//Write file-path expression to process input
-		bgor.inputWriter.Write([]byte(":" + path + "\n"))
+		bgor.inputWriter.Write(gitExpression)
 		bgor.inputWriter.Flush()
 
 		//Read line containing filesize from process output
@@ -97,13 +131,13 @@ func (bgor *BatchGitObjectReader) doCatFile(pathChan chan (string), resultsChan
 			resultsChan <- gitCatFileReadResult{[]byte{}, err}
 			continue
 		}
-		logrus.Debugf("Git Batch Reader: FilePath: %v, Size:%v", path, filesize)
+		logrus.Debugf("Git Batch Reader: FilePath: %v, Size:%v", gitExpression, filesize)
 
 		//Read file contents upto filesize bytes from process output
 		fileBytes := make([]byte, filesize)
 		n, err := io.ReadFull(bgor.outputReader, fileBytes)
 		if n != filesize || err != nil {
-			logrus.Errorf("error reading exactly %v bytes of %v: %v (read %v bytes)", filesize, path, err, n)
+			logrus.Errorf("error reading exactly %v bytes of %v: %v (read %v bytes)", filesize, gitExpression, err, n)
 			resultsChan <- gitCatFileReadResult{[]byte{}, err}
 			continue
 		}