Add option to compute http BodySHA256 on decoded BodyText

lib/http/response.go

@@ -60,6 +60,9 @@ type Response struct {
 	Body       io.ReadCloser   `json:"-"`
 	BodyText   string          `json:"body,omitempty"`
 	BodySHA256 PageFingerprint `json:"body_sha256,omitempty"`
+	// BodyHash is the hash digest hex of the decoded http body, formatted `<kind>:<hex>`
+	// e.g. `sha256:deadbeef100020003000400050006000700080009000a000b000c000d000e000`
+	BodyHash string `json:"body_hash,omitempty"`
 
 	// ContentLength records the length of the associated content. The
 	// value -1 indicates that the length is unknown. Unless Request.Method

modules/http/scanner.go

@@ -9,8 +9,11 @@ package http
 import (
 	"bytes"
 	"context"
+	"crypto/sha1"
 	"crypto/sha256"
+	"encoding/hex"
 	"errors"
+	"fmt"
 	"io"
 	"net"
 	"net/url"
@@ -60,6 +63,10 @@ type Flags struct {
 	RedirectsSucceed bool `long:"redirects-succeed" description:"Redirects are always a success, even if max-redirects is exceeded"`
 
 	OverrideSH bool `long:"override-sig-hash" description:"Override the default SignatureAndHashes TLS option with more expansive default"`
+
+	// ComputeDecodedBodyHashAlgorithm enables computing the body hash later than the default,
+	// using the specified algorithm, allowing a user of the response to recompute a matching hash
+	ComputeDecodedBodyHashAlgorithm string `long:"compute-decoded-body-hash-algorithm" choice:"sha256" choice:"sha1" description:"Choose algorithm for BodyHash field"`
 }
 
 // A Results object is returned by the HTTP module's Scanner.Scan()
@@ -79,7 +86,8 @@ type Module struct {
 
 // Scanner is the implementation of the zgrab2.Scanner interface.
 type Scanner struct {
-	config *Flags
+	config        *Flags
+	decodedHashFn func([]byte) string
 }
 
 // scan holds the state for a single scan. This may entail multiple connections.
@@ -129,6 +137,21 @@ func (s *Scanner) Protocol() string {
 func (scanner *Scanner) Init(flags zgrab2.ScanFlags) error {
 	fl, _ := flags.(*Flags)
 	scanner.config = fl
+
+	if fl.ComputeDecodedBodyHashAlgorithm == "sha1" {
+		scanner.decodedHashFn = func(body []byte) string {
+			raw_hash := sha1.Sum(body)
+			return fmt.Sprintf("sha1:%s", hex.EncodeToString(raw_hash[:]))
+		}
+	} else if fl.ComputeDecodedBodyHashAlgorithm == "sha256" {
+		scanner.decodedHashFn = func(body []byte) string {
+			raw_hash := sha256.Sum256(body)
+			return fmt.Sprintf("sha256:%s", hex.EncodeToString(raw_hash[:]))
+		}
+	} else if fl.ComputeDecodedBodyHashAlgorithm != "" {
+		log.Panicf("Invalid ComputeDecodedBodyHashAlgorithm choice made it through zflags: %s", scanner.config.ComputeDecodedBodyHashAlgorithm)
+	}
+
 	return nil
 }
 
@@ -410,9 +433,13 @@ func (scan *scan) Grab() *zgrab2.ScanError {
 	}
 
 	if len(scan.results.Response.BodyText) > 0 {
-		m := sha256.New()
-		m.Write(buf.Bytes())
-		scan.results.Response.BodySHA256 = m.Sum(nil)
+		if scan.scanner.decodedHashFn != nil {
+			scan.results.Response.BodyHash = scan.scanner.decodedHashFn([]byte(scan.results.Response.BodyText))
+		} else {
+			m := sha256.New()
+			m.Write(buf.Bytes())
+			scan.results.Response.BodySHA256 = m.Sum(nil)
+		}
 	}
 
 	return nil