feat(schema): Add schema generation (#32)

* feat(schema): Add schema generation

* mv pkg/rego and pkg/rules from trivy-policies

* update go mod

* mv internal/rules/ pkg

* update schema ids

* remove trivy-policies/pkg/rego imports

* fix schema json

* fix cyclic dependency

* fix Test_OS_FS

* chore: bump defsec and trivy-policies (#33)

* bump defsec and trivy-policies

* test: use embedded FS

* refactor: use embedded FS to generate documentation

* chore: bump defsec and trivy-policies

* refactor: use registered rules as is

---------

Co-authored-by: Nikita Pivkin <nikita.pivkin@smartforce.io>

.github/workflows/verify-schema.yml

@@ -0,0 +1,19 @@
+name: verify schema
+on:
+  pull_request:
+  merge_group:
+jobs:
+  build:
+    name: verifying schema
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/setup-go@v4
+      with:
+        go-version-file: go.mod
+        cache: true
+        cache-dependency-path: go.sum
+
+    - run: go run ./cmd/schema verify

Makefile

@@ -10,4 +10,21 @@ quality:
 .PHONY: update-aws-deps
 update-aws-deps:
 	@grep aws-sdk-go-v2 go.mod | grep -v '// indirect' | sed 's/^[\t\s]*//g' | sed 's/\s.*//g' | xargs go get
-	@go mod tidy
+	@go mod tidy
+
+.PHONY: schema
+schema:
+	go run ./cmd/schema generate
+
+.PHONY: docs
+docs:
+	go run ./cmd/avd_generator
+
+.PHONY: docs-test
+docs-test:
+	go test -v ./cmd/avd_generator/...
+
+.PHONY: id
+id:
+	@go run ./cmd/id
+

cmd/avd_generator/main.go

@@ -0,0 +1,194 @@
+package main
+
+import (
+	"fmt"
+	goast "go/ast"
+	"go/parser"
+	"go/token"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/aquasecurity/defsec/pkg/framework"
+	"github.com/aquasecurity/trivy-policies/rules"
+
+	_ "github.com/aquasecurity/trivy-iac/pkg/rego"
+	registered "github.com/aquasecurity/trivy-iac/pkg/rules"
+	"github.com/aquasecurity/trivy-iac/pkg/types"
+)
+
+func main() {
+	var generateCount int
+
+	for _, metadata := range registered.GetRegistered(framework.ALL) {
+		writeDocsFile(metadata, "avd_docs")
+		generateCount++
+	}
+
+	fmt.Printf("\nGenerated %d files in avd_docs\n", generateCount)
+}
+
+// nolint: cyclop
+func writeDocsFile(meta types.RegisteredRule, path string) {
+
+	tmpl, err := template.New("defsec").Parse(docsMarkdownTemplate)
+	if err != nil {
+		fail("error occurred creating the template %v\n", err)
+	}
+
+	docpath := filepath.Join(path,
+		strings.ToLower(meta.GetRule().Provider.ConstName()),
+		strings.ToLower(strings.ReplaceAll(meta.GetRule().Service, "-", "")),
+		meta.GetRule().AVDID,
+	)
+
+	if err := os.MkdirAll(docpath, os.ModePerm); err != nil {
+		panic(err)
+	}
+
+	file, err := os.Create(filepath.Join(docpath, "docs.md"))
+	if err != nil {
+		fail("error occurred creating the docs file for %s", docpath)
+	}
+
+	if err := tmpl.Execute(file, meta.GetRule()); err != nil {
+		fail("error occurred generating the document %v", err)
+	}
+	fmt.Printf("Generating docs file for policy %s\n", meta.GetRule().AVDID)
+
+	if meta.GetRule().Terraform != nil {
+		if len(meta.GetRule().Terraform.GoodExamples) > 0 || len(meta.GetRule().Terraform.Links) > 0 {
+			if meta.GetRule().RegoPackage != "" { // get examples from file as rego rules don't have embedded
+				value, err := GetExampleValueFromFile(meta.GetRule().Terraform.GoodExamples[0], "GoodExamples")
+				if err != nil {
+					fail("error retrieving examples from metadata: %v\n", err)
+				}
+				meta.GetRule().Terraform.GoodExamples = []string{value}
+			}
+
+			tmpl, err := template.New("terraform").Parse(terraformMarkdownTemplate)
+			if err != nil {
+				fail("error occurred creating the template %v\n", err)
+			}
+			file, err := os.Create(filepath.Join(docpath, "Terraform.md"))
+			if err != nil {
+				fail("error occurred creating the Terraform file for %s", docpath)
+			}
+			defer func() { _ = file.Close() }()
+
+			if err := tmpl.Execute(file, meta.GetRule()); err != nil {
+				fail("error occurred generating the document %v", err)
+			}
+			fmt.Printf("Generating Terraform file for policy %s\n", meta.GetRule().AVDID)
+		}
+	}
+
+	if meta.GetRule().CloudFormation != nil {
+		if len(meta.GetRule().CloudFormation.GoodExamples) > 0 || len(meta.GetRule().CloudFormation.Links) > 0 {
+			if meta.GetRule().RegoPackage != "" { // get examples from file as rego rules don't have embedded
+				value, err := GetExampleValueFromFile(meta.GetRule().CloudFormation.GoodExamples[0], "GoodExamples")
+				if err != nil {
+					fail("error retrieving examples from metadata: %v\n", err)
+				}
+				meta.GetRule().CloudFormation.GoodExamples = []string{value}
+			}
+
+			tmpl, err := template.New("cloudformation").Parse(cloudformationMarkdownTemplate)
+			if err != nil {
+				fail("error occurred creating the template %v\n", err)
+			}
+			file, err := os.Create(filepath.Join(docpath, "CloudFormation.md"))
+			if err != nil {
+				fail("error occurred creating the CloudFormation file for %s", docpath)
+			}
+			defer func() { _ = file.Close() }()
+
+			if err := tmpl.Execute(file, meta.GetRule()); err != nil {
+				fail("error occurred generating the document %v", err)
+			}
+			fmt.Printf("Generating CloudFormation file for policy %s\n", meta.GetRule().AVDID)
+		}
+	}
+}
+
+func fail(msg string, args ...interface{}) {
+	fmt.Printf(msg, args...)
+	os.Exit(1)
+}
+
+func readFileFromPolicyFS(path string) (io.Reader, error) {
+	path = strings.TrimPrefix(path, "rules/")
+	return rules.EmbeddedPolicyFileSystem.Open(path)
+
+}
+
+func GetExampleValueFromFile(filename string, exampleType string) (string, error) {
+	r, err := readFileFromPolicyFS(filename)
+	if err != nil {
+		return "", err
+	}
+	f, err := parser.ParseFile(token.NewFileSet(), filename, r, parser.AllErrors)
+	if err != nil {
+		return "", err
+	}
+
+	for _, d := range f.Decls {
+		switch decl := d.(type) {
+		case *goast.GenDecl:
+			for _, spec := range decl.Specs {
+				switch spec := spec.(type) {
+				case *goast.ValueSpec:
+					for _, id := range spec.Names {
+						switch v := id.Obj.Decl.(*goast.ValueSpec).Values[0].(type) {
+						case *goast.CompositeLit:
+							value := v.Elts[0].(*goast.BasicLit).Value
+							if strings.Contains(id.Name, exampleType) {
+								return strings.ReplaceAll(value, "`", ""), nil
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	return "", fmt.Errorf("exampleType %s not found in file: %s", exampleType, filename)
+}
+
+var docsMarkdownTemplate = `
+{{ .Explanation }}
+
+### Impact
+{{ if .Impact }}{{ .Impact }}{{ else }}<!-- Add Impact here -->{{ end }}
+
+<!-- DO NOT CHANGE -->
+{{ ` + "`{{ " + `remediationActions ` + "`}}" + `}}
+
+{{ if .Links }}### Links{{ range .Links }}
+- {{ . }}
+{{ end}}
+{{ end }}
+`
+
+var terraformMarkdownTemplate = `
+{{ .Resolution }}
+
+{{ if .Terraform.GoodExamples }}{{ range .Terraform.GoodExamples }}` + "```hcl" + `{{ . }}
+` + "```" + `
+{{ end}}{{ end }}
+{{ if .Terraform.Links }}#### Remediation Links{{ range .Terraform.Links }}
+ - {{ . }}
+{{ end}}{{ end }}
+`
+
+var cloudformationMarkdownTemplate = `
+{{ .Resolution }}
+
+{{ if .CloudFormation.GoodExamples }}{{ range .CloudFormation.GoodExamples }}` + "```yaml" + `{{ . }}
+` + "```" + `
+{{ end}}{{ end }}
+{{ if .CloudFormation.Links }}#### Remediation Links{{ range .CloudFormation.Links }}
+ - {{ . }}
+{{ end}}{{ end }}
+`

cmd/avd_generator/main_test.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/aquasecurity/defsec/pkg/framework"
+	registered "github.com/aquasecurity/trivy-iac/pkg/rules"
+)
+
+func init() { // change the pwd for the test to top level defesc dir
+	_, filename, _, _ := runtime.Caller(0)
+	dir := path.Join(path.Dir(filename), "../..")
+	err := os.Chdir(dir)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func Test_AVDPageGeneration(t *testing.T) {
+	tmpDir := t.TempDir()
+	defer func() {
+		os.RemoveAll(tmpDir)
+	}()
+
+	var generateCount int
+	for _, metadata := range registered.GetRegistered(framework.ALL) {
+		writeDocsFile(metadata, tmpDir)
+		generateCount++
+	}
+	fmt.Printf("\nGenerated %d files in avd_docs\n", generateCount)
+
+	// check golang policies
+	b, err := os.ReadFile(filepath.Join(tmpDir, "aws/rds/AVD-AWS-0077", "Terraform.md"))
+	require.NoError(t, err)
+	assert.Contains(t, string(b), `hcl
+ resource "aws_rds_cluster" "good_example" {
+ 	cluster_identifier      = "aurora-cluster-demo"
+ 	engine                  = "aurora-mysql"
+ 	engine_version          = "5.7.mysql_aurora.2.03.2"
+ 	availability_zones      = ["us-west-2a", "us-west-2b", "us-west-2c"]
+ 	database_name           = "mydb"
+ 	master_username         = "foo"
+ 	master_password         = "bar"
+ 	backup_retention_period = 5
+ 	preferred_backup_window = "07:00-09:00"
+   }`)
+
+	b, err = os.ReadFile(filepath.Join(tmpDir, "aws/rds/AVD-AWS-0077", "CloudFormation.md"))
+	require.NoError(t, err)
+	assert.Contains(t, string(b), `yaml---
+AWSTemplateFormatVersion: 2010-09-09
+Description: Good example
+Resources:
+  Queue:
+    Type: AWS::RDS::DBInstance
+    Properties:
+      BackupRetentionPeriod: 30
+`)
+
+	// check rego policies
+	b, err = os.ReadFile(filepath.Join(tmpDir, "aws/rds/AVD-AWS-0180", "Terraform.md"))
+	require.NoError(t, err)
+	assert.Contains(t, string(b), `hcl
+ resource "aws_db_instance" "good_example" {
+ 	publicly_accessible = false
+ }`)
+
+	b, err = os.ReadFile(filepath.Join(tmpDir, "aws/rds/AVD-AWS-0180", "CloudFormation.md"))
+	require.NoError(t, err)
+	assert.Contains(t, string(b), `yaml---
+AWSTemplateFormatVersion: 2010-09-09
+Description: Good example
+Resources:
+  Queue:
+    Type: AWS::RDS::DBInstance
+    Properties:
+      PubliclyAccessible: false`)
+}

cmd/id/main.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/aquasecurity/defsec/pkg/framework"
+
+	_ "github.com/aquasecurity/trivy-iac/pkg/rego"
+	"github.com/aquasecurity/trivy-iac/pkg/rules"
+)
+
+func main() {
+
+	// organise existing rules by provider
+	keyMap := make(map[string][]string)
+	for _, rule := range rules.GetRegistered(framework.ALL) {
+		id := rule.GetRule().AVDID
+		if id == "" {
+			continue
+		}
+		parts := strings.Split(id, "-")
+		if len(parts) != 3 {
+			continue
+		}
+		keyMap[parts[1]] = append(keyMap[parts[1]], parts[2])
+	}
+
+	fmt.Print("\nThe following IDs are free - choose the one for the service you are targeting.\n\n")
+
+	var freeIDs []string
+	for key := range keyMap {
+		sort.Strings(keyMap[key])
+		all := keyMap[key]
+		max := all[len(all)-1]
+		i, err := strconv.Atoi(max)
+		if err != nil {
+			_, _ = fmt.Fprintf(os.Stderr, "Error, invalid AVD ID: AVD-%s-%s\n", key, max)
+		}
+		free := fmt.Sprintf("AVD-%s-%04d", key, i+1)
+		freeIDs = append(freeIDs, fmt.Sprintf("%16s: %s", key, free))
+	}
+
+	sort.Slice(freeIDs, func(i, j int) bool {
+		return strings.TrimSpace(freeIDs[i]) < strings.TrimSpace(freeIDs[j])
+	})
+	fmt.Println(strings.Join(freeIDs, "\n"))
+
+}