chore: bump defsec and trivy-policies (#33)

* bump defsec and trivy-policies

* test: use embedded FS

* refactor: use embedded FS to generate documentation

* chore: bump defsec and trivy-policies

* refactor: use registered rules as is

Makefile

@@ -16,3 +16,15 @@ update-aws-deps:
 schema:
 	go run ./cmd/schema generate
 
+.PHONY: docs
+docs:
+	go run ./cmd/avd_generator
+
+.PHONY: docs-test
+docs-test:
+	go test -v ./cmd/avd_generator/...
+
+.PHONY: id
+id:
+	@go run ./cmd/id
+

cmd/avd_generator/main.go

@@ -0,0 +1,194 @@
+package main
+
+import (
+	"fmt"
+	goast "go/ast"
+	"go/parser"
+	"go/token"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/aquasecurity/defsec/pkg/framework"
+	"github.com/aquasecurity/trivy-policies/rules"
+
+	_ "github.com/aquasecurity/trivy-iac/pkg/rego"
+	registered "github.com/aquasecurity/trivy-iac/pkg/rules"
+	"github.com/aquasecurity/trivy-iac/pkg/types"
+)
+
+func main() {
+	var generateCount int
+
+	for _, metadata := range registered.GetRegistered(framework.ALL) {
+		writeDocsFile(metadata, "avd_docs")
+		generateCount++
+	}
+
+	fmt.Printf("\nGenerated %d files in avd_docs\n", generateCount)
+}
+
+// nolint: cyclop
+func writeDocsFile(meta types.RegisteredRule, path string) {
+
+	tmpl, err := template.New("defsec").Parse(docsMarkdownTemplate)
+	if err != nil {
+		fail("error occurred creating the template %v\n", err)
+	}
+
+	docpath := filepath.Join(path,
+		strings.ToLower(meta.GetRule().Provider.ConstName()),
+		strings.ToLower(strings.ReplaceAll(meta.GetRule().Service, "-", "")),
+		meta.GetRule().AVDID,
+	)
+
+	if err := os.MkdirAll(docpath, os.ModePerm); err != nil {
+		panic(err)
+	}
+
+	file, err := os.Create(filepath.Join(docpath, "docs.md"))
+	if err != nil {
+		fail("error occurred creating the docs file for %s", docpath)
+	}
+
+	if err := tmpl.Execute(file, meta.GetRule()); err != nil {
+		fail("error occurred generating the document %v", err)
+	}
+	fmt.Printf("Generating docs file for policy %s\n", meta.GetRule().AVDID)
+
+	if meta.GetRule().Terraform != nil {
+		if len(meta.GetRule().Terraform.GoodExamples) > 0 || len(meta.GetRule().Terraform.Links) > 0 {
+			if meta.GetRule().RegoPackage != "" { // get examples from file as rego rules don't have embedded
+				value, err := GetExampleValueFromFile(meta.GetRule().Terraform.GoodExamples[0], "GoodExamples")
+				if err != nil {
+					fail("error retrieving examples from metadata: %v\n", err)
+				}
+				meta.GetRule().Terraform.GoodExamples = []string{value}
+			}
+
+			tmpl, err := template.New("terraform").Parse(terraformMarkdownTemplate)
+			if err != nil {
+				fail("error occurred creating the template %v\n", err)
+			}
+			file, err := os.Create(filepath.Join(docpath, "Terraform.md"))
+			if err != nil {
+				fail("error occurred creating the Terraform file for %s", docpath)
+			}
+			defer func() { _ = file.Close() }()
+
+			if err := tmpl.Execute(file, meta.GetRule()); err != nil {
+				fail("error occurred generating the document %v", err)
+			}
+			fmt.Printf("Generating Terraform file for policy %s\n", meta.GetRule().AVDID)
+		}
+	}
+
+	if meta.GetRule().CloudFormation != nil {
+		if len(meta.GetRule().CloudFormation.GoodExamples) > 0 || len(meta.GetRule().CloudFormation.Links) > 0 {
+			if meta.GetRule().RegoPackage != "" { // get examples from file as rego rules don't have embedded
+				value, err := GetExampleValueFromFile(meta.GetRule().CloudFormation.GoodExamples[0], "GoodExamples")
+				if err != nil {
+					fail("error retrieving examples from metadata: %v\n", err)
+				}
+				meta.GetRule().CloudFormation.GoodExamples = []string{value}
+			}
+
+			tmpl, err := template.New("cloudformation").Parse(cloudformationMarkdownTemplate)
+			if err != nil {
+				fail("error occurred creating the template %v\n", err)
+			}
+			file, err := os.Create(filepath.Join(docpath, "CloudFormation.md"))
+			if err != nil {
+				fail("error occurred creating the CloudFormation file for %s", docpath)
+			}
+			defer func() { _ = file.Close() }()
+
+			if err := tmpl.Execute(file, meta.GetRule()); err != nil {
+				fail("error occurred generating the document %v", err)
+			}
+			fmt.Printf("Generating CloudFormation file for policy %s\n", meta.GetRule().AVDID)
+		}
+	}
+}
+
+func fail(msg string, args ...interface{}) {
+	fmt.Printf(msg, args...)
+	os.Exit(1)
+}
+
+func readFileFromPolicyFS(path string) (io.Reader, error) {
+	path = strings.TrimPrefix(path, "rules/")
+	return rules.EmbeddedPolicyFileSystem.Open(path)
+
+}
+
+func GetExampleValueFromFile(filename string, exampleType string) (string, error) {
+	r, err := readFileFromPolicyFS(filename)
+	if err != nil {
+		return "", err
+	}
+	f, err := parser.ParseFile(token.NewFileSet(), filename, r, parser.AllErrors)
+	if err != nil {
+		return "", err
+	}
+
+	for _, d := range f.Decls {
+		switch decl := d.(type) {
+		case *goast.GenDecl:
+			for _, spec := range decl.Specs {
+				switch spec := spec.(type) {
+				case *goast.ValueSpec:
+					for _, id := range spec.Names {
+						switch v := id.Obj.Decl.(*goast.ValueSpec).Values[0].(type) {
+						case *goast.CompositeLit:
+							value := v.Elts[0].(*goast.BasicLit).Value
+							if strings.Contains(id.Name, exampleType) {
+								return strings.ReplaceAll(value, "`", ""), nil
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	return "", fmt.Errorf("exampleType %s not found in file: %s", exampleType, filename)
+}
+
+var docsMarkdownTemplate = `
+{{ .Explanation }}
+
+### Impact
+{{ if .Impact }}{{ .Impact }}{{ else }}<!-- Add Impact here -->{{ end }}
+
+<!-- DO NOT CHANGE -->
+{{ ` + "`{{ " + `remediationActions ` + "`}}" + `}}
+
+{{ if .Links }}### Links{{ range .Links }}
+- {{ . }}
+{{ end}}
+{{ end }}
+`
+
+var terraformMarkdownTemplate = `
+{{ .Resolution }}
+
+{{ if .Terraform.GoodExamples }}{{ range .Terraform.GoodExamples }}` + "```hcl" + `{{ . }}
+` + "```" + `
+{{ end}}{{ end }}
+{{ if .Terraform.Links }}#### Remediation Links{{ range .Terraform.Links }}
+ - {{ . }}
+{{ end}}{{ end }}
+`
+
+var cloudformationMarkdownTemplate = `
+{{ .Resolution }}
+
+{{ if .CloudFormation.GoodExamples }}{{ range .CloudFormation.GoodExamples }}` + "```yaml" + `{{ . }}
+` + "```" + `
+{{ end}}{{ end }}
+{{ if .CloudFormation.Links }}#### Remediation Links{{ range .CloudFormation.Links }}
+ - {{ . }}
+{{ end}}{{ end }}
+`

cmd/avd_generator/main_test.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/aquasecurity/defsec/pkg/framework"
+	registered "github.com/aquasecurity/trivy-iac/pkg/rules"
+)
+
+func init() { // change the pwd for the test to top level defesc dir
+	_, filename, _, _ := runtime.Caller(0)
+	dir := path.Join(path.Dir(filename), "../..")
+	err := os.Chdir(dir)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func Test_AVDPageGeneration(t *testing.T) {
+	tmpDir := t.TempDir()
+	defer func() {
+		os.RemoveAll(tmpDir)
+	}()
+
+	var generateCount int
+	for _, metadata := range registered.GetRegistered(framework.ALL) {
+		writeDocsFile(metadata, tmpDir)
+		generateCount++
+	}
+	fmt.Printf("\nGenerated %d files in avd_docs\n", generateCount)
+
+	// check golang policies
+	b, err := os.ReadFile(filepath.Join(tmpDir, "aws/rds/AVD-AWS-0077", "Terraform.md"))
+	require.NoError(t, err)
+	assert.Contains(t, string(b), `hcl
+ resource "aws_rds_cluster" "good_example" {
+ 	cluster_identifier      = "aurora-cluster-demo"
+ 	engine                  = "aurora-mysql"
+ 	engine_version          = "5.7.mysql_aurora.2.03.2"
+ 	availability_zones      = ["us-west-2a", "us-west-2b", "us-west-2c"]
+ 	database_name           = "mydb"
+ 	master_username         = "foo"
+ 	master_password         = "bar"
+ 	backup_retention_period = 5
+ 	preferred_backup_window = "07:00-09:00"
+   }`)
+
+	b, err = os.ReadFile(filepath.Join(tmpDir, "aws/rds/AVD-AWS-0077", "CloudFormation.md"))
+	require.NoError(t, err)
+	assert.Contains(t, string(b), `yaml---
+AWSTemplateFormatVersion: 2010-09-09
+Description: Good example
+Resources:
+  Queue:
+    Type: AWS::RDS::DBInstance
+    Properties:
+      BackupRetentionPeriod: 30
+`)
+
+	// check rego policies
+	b, err = os.ReadFile(filepath.Join(tmpDir, "aws/rds/AVD-AWS-0180", "Terraform.md"))
+	require.NoError(t, err)
+	assert.Contains(t, string(b), `hcl
+ resource "aws_db_instance" "good_example" {
+ 	publicly_accessible = false
+ }`)
+
+	b, err = os.ReadFile(filepath.Join(tmpDir, "aws/rds/AVD-AWS-0180", "CloudFormation.md"))
+	require.NoError(t, err)
+	assert.Contains(t, string(b), `yaml---
+AWSTemplateFormatVersion: 2010-09-09
+Description: Good example
+Resources:
+  Queue:
+    Type: AWS::RDS::DBInstance
+    Properties:
+      PubliclyAccessible: false`)
+}

cmd/id/main.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/aquasecurity/defsec/pkg/framework"
+
+	_ "github.com/aquasecurity/trivy-iac/pkg/rego"
+	"github.com/aquasecurity/trivy-iac/pkg/rules"
+)
+
+func main() {
+
+	// organise existing rules by provider
+	keyMap := make(map[string][]string)
+	for _, rule := range rules.GetRegistered(framework.ALL) {
+		id := rule.GetRule().AVDID
+		if id == "" {
+			continue
+		}
+		parts := strings.Split(id, "-")
+		if len(parts) != 3 {
+			continue
+		}
+		keyMap[parts[1]] = append(keyMap[parts[1]], parts[2])
+	}
+
+	fmt.Print("\nThe following IDs are free - choose the one for the service you are targeting.\n\n")
+
+	var freeIDs []string
+	for key := range keyMap {
+		sort.Strings(keyMap[key])
+		all := keyMap[key]
+		max := all[len(all)-1]
+		i, err := strconv.Atoi(max)
+		if err != nil {
+			_, _ = fmt.Fprintf(os.Stderr, "Error, invalid AVD ID: AVD-%s-%s\n", key, max)
+		}
+		free := fmt.Sprintf("AVD-%s-%04d", key, i+1)
+		freeIDs = append(freeIDs, fmt.Sprintf("%16s: %s", key, free))
+	}
+
+	sort.Slice(freeIDs, func(i, j int) bool {
+		return strings.TrimSpace(freeIDs[i]) < strings.TrimSpace(freeIDs[j])
+	})
+	fmt.Println(strings.Join(freeIDs, "\n"))
+
+}

go.mod

@@ -7,8 +7,8 @@ require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/alecthomas/chroma v0.10.0 // indirect
 	github.com/apparentlymart/go-cidr v1.1.0
-	github.com/aquasecurity/defsec v0.93.2-0.20231010220704-82e20e4c9d72
-	github.com/aquasecurity/trivy-policies v0.2.1-0.20231017063459-bd8eb4f035d0
+	github.com/aquasecurity/defsec v0.93.2-0.20231020041402-7ccc46780c09
+	github.com/aquasecurity/trivy-policies v0.3.1-0.20231021040354-0572a07131c2
 	github.com/aws/aws-sdk-go v1.44.245 // indirect
 	github.com/aws/smithy-go v1.14.2
 	github.com/bmatcuk/doublestar/v4 v4.6.0

go.sum

@@ -238,10 +238,10 @@ github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6
 github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
 github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
 github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
-github.com/aquasecurity/defsec v0.93.2-0.20231010220704-82e20e4c9d72 h1:myA/gpe55MGv88FHxN3YCchrrUM5ze4d+MQ42FeunVw=
-github.com/aquasecurity/defsec v0.93.2-0.20231010220704-82e20e4c9d72/go.mod h1:hSw5Nz91TC34eUmgXrpZSzI6zd7hErpk1QyE+uqtBjc=
-github.com/aquasecurity/trivy-policies v0.2.1-0.20231017063459-bd8eb4f035d0 h1:DaUmplSzs3o61bPVw+TyEF4mXyx7fmLfFORhep/QIBE=
-github.com/aquasecurity/trivy-policies v0.2.1-0.20231017063459-bd8eb4f035d0/go.mod h1:95+qxzR/aSisozbOeSHtltIstUhDHzOCeaqYOFQhC80=
+github.com/aquasecurity/defsec v0.93.2-0.20231020041402-7ccc46780c09 h1:dYBDwBnNzDsJr6l+FkrkrvWysAKc6VAO/leOcjvJfaA=
+github.com/aquasecurity/defsec v0.93.2-0.20231020041402-7ccc46780c09/go.mod h1:J30VViSgmoW2Ic/6aqVJO2qvuADsmZ3MYuNxPcU6Vt0=
+github.com/aquasecurity/trivy-policies v0.3.1-0.20231021040354-0572a07131c2 h1:Xkm2i9Dy98p/DMR0smfog487zaTJ11hLVL+PvIgVWyM=
+github.com/aquasecurity/trivy-policies v0.3.1-0.20231021040354-0572a07131c2/go.mod h1:Wqj81EIp4lDQGVzbPalKLNucR7c96YLQbfdA60KpEkQ=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=