From b4400e540e76c966535fc63320a4e71a12f57d5f Mon Sep 17 00:00:00 2001 From: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> Date: Thu, 14 Nov 2024 14:27:01 -0600 Subject: [PATCH 1/7] Refactor the cmd for custom - refactored the custom cmd Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> --- cmd/leaderboard/leaderboard.go | 3 +- cmd/query/custom/custom.go | 142 ++++++-------- cmd/query/custom/custom_test.go | 315 ++++++++++++++++++++++++++++++++ cmd/query/query.go | 12 +- cmd/root/root.go | 4 +- 5 files changed, 384 insertions(+), 92 deletions(-) create mode 100644 cmd/query/custom/custom_test.go diff --git a/cmd/leaderboard/leaderboard.go b/cmd/leaderboard/leaderboard.go index 8f479dd..a8d6fde 100644 --- a/cmd/leaderboard/leaderboard.go +++ b/cmd/leaderboard/leaderboard.go @@ -3,7 +3,6 @@ package leaderboard import ( "github.com/bitbomdev/minefield/cmd/leaderboard/custom" "github.com/bitbomdev/minefield/cmd/leaderboard/keys" - "github.com/bitbomdev/minefield/pkg/graph" "github.com/spf13/cobra" ) @@ -11,7 +10,7 @@ type options struct{} func (o *options) AddFlags(_ *cobra.Command) {} -func New(storage graph.Storage) *cobra.Command { +func New() *cobra.Command { o := &options{} cmd := &cobra.Command{ Use: "leaderboard", diff --git a/cmd/query/custom/custom.go b/cmd/query/custom/custom.go index 59f37c2..dc6a679 100644 --- a/cmd/query/custom/custom.go +++ b/cmd/query/custom/custom.go @@ -1,157 +1,133 @@ package custom import ( - "bufio" "fmt" + "io" "net/http" - "os" "strconv" "strings" "connectrpc.com/connect" - "github.com/RoaringBitmap/roaring" "github.com/bitbomdev/minefield/cmd/helpers" apiv1 "github.com/bitbomdev/minefield/gen/api/v1" "github.com/bitbomdev/minefield/gen/api/v1/apiv1connect" - "github.com/bitbomdev/minefield/pkg/graph" "github.com/olekukonko/tablewriter" "github.com/spf13/cobra" ) +// options holds the command-line options. type options struct { - storage graph.Storage - visualize bool - visualizerAddr string - maxOutput int - showInfo bool - saveQuery string + maxOutput int + showInfo bool + saveQuery string + addr string + output string + queryServiceClient apiv1connect.QueryServiceClient } +// AddFlags adds command-line flags to the provided cobra command. func (o *options) AddFlags(cmd *cobra.Command) { - cmd.Flags().IntVar(&o.maxOutput, "max-getMetadata", 10, "max getMetadata length") - cmd.Flags().BoolVar(&o.visualize, "visualize", false, "visualize the query") - cmd.Flags().StringVar(&o.visualizerAddr, "addr", "8081", "address to run the visualizer on") + cmd.Flags().IntVar(&o.maxOutput, "max-output", 10, "maximum number of results to display") cmd.Flags().BoolVar(&o.showInfo, "show-info", true, "display the info column") - cmd.Flags().StringVar(&o.saveQuery, "save-query", "", "save the query to a specific file") + cmd.Flags().StringVar(&o.addr, "addr", "http://localhost:8089", "address of the minefield server") + cmd.Flags().StringVar(&o.output, "output", "table", "output format (table or json)") } +// Run executes the custom command with the provided arguments. func (o *options) Run(cmd *cobra.Command, args []string) error { script := strings.Join(args, " ") - if strings.TrimSpace(script) == "" { return fmt.Errorf("script cannot be empty") } - httpClient := &http.Client{} - addr := os.Getenv("BITBOMDEV_ADDR") - if addr == "" { - addr = "http://localhost:8089" + + // Initialize client if not injected (for testing) + if o.queryServiceClient == nil { + o.queryServiceClient = apiv1connect.NewQueryServiceClient( + http.DefaultClient, + o.addr, + connect.WithGRPC(), + connect.WithSendGzip(), + ) } - client := apiv1connect.NewQueryServiceClient(httpClient, addr) - // Create a new context ctx := cmd.Context() - - // Create a new QueryRequest req := connect.NewRequest(&apiv1.QueryRequest{ Script: script, }) - // Make the Query request - res, err := client.Query(ctx, req) + res, err := o.queryServiceClient.Query(ctx, req) if err != nil { return fmt.Errorf("query failed: %v", err) } - // Initialize the table - table := tablewriter.NewWriter(os.Stdout) - table.SetAutoWrapText(false) - table.SetRowLine(true) + if len(res.Msg.Nodes) == 0 { + return fmt.Errorf("no nodes found for script: %s", script) + } - // Dynamically set the header based on the showInfo flag + switch o.output { + case "json": + jsonOutput, err := helpers.FormatNodeJSON(res.Msg.Nodes) + if err != nil { + return fmt.Errorf("failed to format nodes as JSON: %w", err) + } + cmd.Println(string(jsonOutput)) + return nil + case "table": + return formatTable(cmd.OutOrStdout(), res.Msg.Nodes, o.maxOutput, o.showInfo) + default: + return fmt.Errorf("unknown output format: %s", o.output) + } +} + +// formatTable formats the nodes into a table and writes it to the provided writer. +func formatTable(w io.Writer, nodes []*apiv1.Node, maxOutput int, showInfo bool) error { + table := tablewriter.NewWriter(w) headers := []string{"Name", "Type", "ID"} - if o.showInfo { + if showInfo { headers = append(headers, "Info") } table.SetHeader(headers) + table.SetAutoWrapText(false) + table.SetRowLine(true) - // Build the rows count := 0 - var f *os.File - if o.saveQuery != "" { - f, err = os.Create(o.saveQuery) - if err != nil { - return err - } - defer f.Close() - } - for _, node := range res.Msg.Nodes { - if count >= o.maxOutput { + for _, node := range nodes { + if count >= maxOutput { break } - // Build the common row data row := []string{ node.Name, node.Type, - strconv.Itoa(int(node.Id)), + strconv.FormatUint(uint64(node.Id), 10), } - // If showInfo is true, compute the additionalInfo and append it - if o.showInfo { + if showInfo { additionalInfo := helpers.ComputeAdditionalInfo(node) row = append(row, additionalInfo) } - // Append the row to the table table.Append(row) - - if o.saveQuery != "" { - f.WriteString(node.Name + "\n") - } count++ } - // Render the table table.Render() - - // Visualization logic (remaining the same) - if o.visualize { - server := &http.Server{ - Addr: ":" + o.visualizerAddr, - } - - ids := roaring.New() - - for _, node := range res.Msg.Nodes { - ids.Add(node.Id) - } - - shutdown, err := graph.RunGraphVisualizer(o.storage, ids, script, server) - if err != nil { - return err - } - defer shutdown() - - fmt.Println("Press Enter to stop the server and continue...") - if _, err := bufio.NewReader(os.Stdin).ReadBytes('\n'); err != nil { - return err - } - } - return nil } -func New(storage graph.Storage) *cobra.Command { - o := &options{ - storage: storage, - } +// New creates and returns a new Cobra command for executing custom query scripts. +func New() *cobra.Command { + o := &options{} + cmd := &cobra.Command{ Use: "custom [script]", - Short: "Query dependencies and dependents of a project", - Args: cobra.MinimumNArgs(1), + Short: "Execute a custom query script", + Long: "Execute a custom query script to perform tailored queries against the project's dependencies and dependents.", + Args: cobra.ExactArgs(1), RunE: o.Run, DisableAutoGenTag: true, } + o.AddFlags(cmd) return cmd diff --git a/cmd/query/custom/custom_test.go b/cmd/query/custom/custom_test.go new file mode 100644 index 0000000..e7c4864 --- /dev/null +++ b/cmd/query/custom/custom_test.go @@ -0,0 +1,315 @@ +package custom + +import ( + "bytes" + "context" + "errors" + "strconv" + "strings" + "testing" + + "connectrpc.com/connect" + apiv1 "github.com/bitbomdev/minefield/gen/api/v1" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" +) + +func TestOptions_AddFlags(t *testing.T) { + o := &options{} + cmd := &cobra.Command{} + + o.AddFlags(cmd) + + // Test "max-output" flag + maxOutputFlag := cmd.Flags().Lookup("max-output") + if maxOutputFlag == nil { + t.Error("Expected 'max-output' flag to be defined") + } else { + if maxOutputFlag.DefValue != "10" { + t.Errorf("Expected default value of 'max-output' to be '10', got '%s'", maxOutputFlag.DefValue) + } + } + + // Test "show-info" flag + showInfoFlag := cmd.Flags().Lookup("show-info") + if showInfoFlag == nil { + t.Error("Expected 'show-info' flag to be defined") + } else { + if showInfoFlag.DefValue != "true" { + t.Errorf("Expected default value of 'show-info' to be 'true', got '%s'", showInfoFlag.DefValue) + } + } + + // Test "addr" flag + addrFlag := cmd.Flags().Lookup("addr") + if addrFlag == nil { + t.Error("Expected 'addr' flag to be defined") + } else { + if addrFlag.DefValue != "http://localhost:8089" { + t.Errorf("Expected default value of 'addr' to be 'http://localhost:8089', got '%s'", addrFlag.DefValue) + } + } + + // Test "output" flag + outputFlag := cmd.Flags().Lookup("output") + if outputFlag == nil { + t.Error("Expected 'output' flag to be defined") + } else { + if outputFlag.DefValue != "table" { + t.Errorf("Expected default value of 'output' to be 'table', got '%s'", outputFlag.DefValue) + } + } +} + +func TestNewCommand(t *testing.T) { + cmd := New() + + if cmd == nil { + t.Fatal("Expected New() to return a non-nil command") + } + + // Check the command's basic properties + if cmd.Use != "custom [script]" { + t.Errorf("Expected Use: 'custom [script]', got: '%s'", cmd.Use) + } + + if cmd.Short != "Execute a custom query script" { + t.Errorf("Expected Short: 'Execute a custom query script', got: '%s'", cmd.Short) + } + + expectedLong := "Execute a custom query script to perform tailored queries against the project's dependencies and dependents." + if cmd.Long != expectedLong { + t.Errorf("Expected Long: '%s', got: '%s'", expectedLong, cmd.Long) + } + + if cmd.Args == nil { + t.Error("Expected Args to be defined") + } + + if cmd.RunE == nil { + t.Error("Expected RunE to be defined") + } + + // Verify that the flags are added to the command + flags := []struct { + name string + shorthand string + defValue string + }{ + {name: "max-output", defValue: "10"}, + {name: "show-info", defValue: "true"}, + {name: "addr", defValue: "http://localhost:8089"}, + {name: "output", defValue: "table"}, + } + + for _, flag := range flags { + f := cmd.Flags().Lookup(flag.name) + if f == nil { + t.Errorf("Expected flag '%s' to be defined", flag.name) + } else { + if f.DefValue != flag.defValue { + t.Errorf("Expected default value of flag '%s' to be '%s', got '%s'", flag.name, flag.defValue, f.DefValue) + } + } + } +} + +// TestFormatTable tests the formatTable function +func TestFormatTable(t *testing.T) { + nodes := []*apiv1.Node{ + { + Name: "Node1", + Type: "TypeA", + Id: 1, + }, + { + Name: "Node2", + Type: "TypeB", + Id: 2, + }, + { + Name: "Node3", + Type: "TypeC", + Id: 3, + }, + } + + t.Run("WithShowInfoTrue", func(t *testing.T) { + var buf bytes.Buffer + err := formatTable(&buf, nodes, 10, true) + assert.NoError(t, err) + + output := buf.String() + + // Check if the output contains expected headers + assert.Contains(t, output, "NAME") + assert.Contains(t, output, "TYPE") + assert.Contains(t, output, "ID") + assert.Contains(t, output, "INFO") + + // Check if the output contains expected node data + for _, node := range nodes { + assert.Contains(t, output, node.Name) + assert.Contains(t, output, node.Type) + assert.Contains(t, output, strconv.FormatUint(uint64(node.Id), 10)) + } + }) + + t.Run("MaxOutputLimit", func(t *testing.T) { + var buf bytes.Buffer + err := formatTable(&buf, nodes, 2, true) + assert.NoError(t, err) + + output := buf.String() + + // Only the first two nodes should be present + assert.Contains(t, output, "Node1") + assert.Contains(t, output, "Node2") + assert.NotContains(t, output, "Node3") + }) +} + +// Mock implementation of QueryServiceClient +type mockQueryServiceClient struct { + QueryFunc func(ctx context.Context, req *connect.Request[apiv1.QueryRequest]) (*connect.Response[apiv1.QueryResponse], error) +} + +func (m *mockQueryServiceClient) Query(ctx context.Context, req *connect.Request[apiv1.QueryRequest]) (*connect.Response[apiv1.QueryResponse], error) { + return m.QueryFunc(ctx, req) +} + +func TestRun(t *testing.T) { + tests := []struct { + name string + script string + output string + maxOutput int + showInfo bool + mockResponse *apiv1.QueryResponse + mockError error + expectError bool + expectedErrorString string + expectedOutput []string + }{ + { + name: "valid response with nodes in json output", + script: "match (n) return n", + output: "json", + maxOutput: 10, + showInfo: true, + mockResponse: &apiv1.QueryResponse{ + Nodes: []*apiv1.Node{ + {Name: "node1", Type: "type1", Id: 1}, + {Name: "node2", Type: "type2", Id: 2}, + }, + }, + expectedOutput: []string{ + `"name": "node1"`, `"type": "type1"`, `"id": "1"`, + `"name": "node2"`, `"type": "type2"`, `"id": "2"`, + }, + }, + { + name: "valid response with nodes in table output", + script: "match (n) return n", + output: "table", + maxOutput: 10, + showInfo: true, + mockResponse: &apiv1.QueryResponse{ + Nodes: []*apiv1.Node{ + {Name: "node1", Type: "type1", Id: 1}, + {Name: "node2", Type: "type2", Id: 2}, + }, + }, + expectedOutput: []string{ + "NAME", "TYPE", "ID", "INFO", + "node1", "type1", "1", + "node2", "type2", "2", + }, + }, + { + name: "no nodes found", + script: "match (n) where n.name = 'unknown' return n", + output: "table", + maxOutput: 10, + showInfo: true, + mockResponse: &apiv1.QueryResponse{Nodes: []*apiv1.Node{}}, + expectError: true, + expectedErrorString: "no nodes found for script: match (n) where n.name = 'unknown' return n", + }, + { + name: "client error", + script: "bad script", + output: "json", + maxOutput: 10, + showInfo: true, + mockError: errors.New("client error"), + expectError: true, + expectedErrorString: "query failed: client error", + }, + { + name: "unknown output format", + script: "match (n) return n", + output: "unknown", + maxOutput: 10, + showInfo: true, + mockResponse: &apiv1.QueryResponse{Nodes: []*apiv1.Node{{Name: "node1", Type: "type1", Id: 1}}}, + expectError: true, + expectedErrorString: "unknown output format: unknown", + }, + { + name: "empty script", + script: " ", + output: "json", + maxOutput: 10, + showInfo: true, + expectError: true, + expectedErrorString: "script cannot be empty", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Mock QueryServiceClient + mockClient := &mockQueryServiceClient{ + QueryFunc: func(ctx context.Context, req *connect.Request[apiv1.QueryRequest]) (*connect.Response[apiv1.QueryResponse], error) { + if tt.mockError != nil { + return nil, tt.mockError + } + return connect.NewResponse(tt.mockResponse), nil + }, + } + + o := &options{ + addr: "http://localhost:8089", + output: tt.output, + maxOutput: tt.maxOutput, + showInfo: tt.showInfo, + queryServiceClient: mockClient, + } + + // Create a cobra command and context + cmd := &cobra.Command{} + cmd.SetOut(new(bytes.Buffer)) // Capture output for assertions + outputBuf := &bytes.Buffer{} + cmd.SetOut(outputBuf) + cmd.SetContext(context.Background()) + + args := []string{tt.script} + + err := o.Run(cmd, args) + + if tt.expectError { + assert.Error(t, err) + assert.Equal(t, tt.expectedErrorString, err.Error()) + } else { + assert.NoError(t, err) + outputStr := outputBuf.String() + for _, expected := range tt.expectedOutput { + if !strings.Contains(outputStr, expected) { + t.Errorf("Output does not contain expected string: %s", expected) + } + } + } + }) + } +} diff --git a/cmd/query/query.go b/cmd/query/query.go index a980b71..3e9fa36 100644 --- a/cmd/query/query.go +++ b/cmd/query/query.go @@ -4,19 +4,21 @@ import ( "github.com/bitbomdev/minefield/cmd/query/custom" "github.com/bitbomdev/minefield/cmd/query/getMetadata" "github.com/bitbomdev/minefield/cmd/query/globsearch" - "github.com/bitbomdev/minefield/pkg/graph" "github.com/spf13/cobra" ) -func New(storage graph.Storage) *cobra.Command { +func New() *cobra.Command { cmd := &cobra.Command{ - Use: "query", - Short: "Query dependencies and dependents of a project", + Use: "query", + Short: "Query dependencies and dependents of a project", + Long: "A comprehensive set of commands to query dependencies and dependents of a project, enabling detailed data retrieval and analysis.", DisableAutoGenTag: true, } - cmd.AddCommand(custom.New(storage)) + // Add subcommands + cmd.AddCommand(custom.New()) cmd.AddCommand(getMetadata.New()) cmd.AddCommand(globsearch.New()) + return cmd } diff --git a/cmd/root/root.go b/cmd/root/root.go index c476bc0..cfdd841 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -31,10 +31,10 @@ func New(storage graph.Storage) *cobra.Command { o.AddFlags(cmd) - cmd.AddCommand(query.New(storage)) + cmd.AddCommand(query.New()) cmd.AddCommand(ingest.New(storage)) cmd.AddCommand(cache.New()) - cmd.AddCommand(leaderboard.New(storage)) + cmd.AddCommand(leaderboard.New()) cmd.AddCommand(server.New(storage)) return cmd From 8f47231c6db33da71a7d3fc0ce6aaf814d66a4aa Mon Sep 17 00:00:00 2001 From: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> Date: Thu, 14 Nov 2024 14:36:58 -0600 Subject: [PATCH 2/7] Removed the storage from cmd that is not server Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> --- api/v1/service_test.go | 2 +- cmd/ingest/ingest.go | 9 ++++----- cmd/ingest/osv/osv.go | 12 ++++-------- cmd/ingest/sbom/sbom.go | 12 ++++-------- cmd/ingest/scorecard/scorecard.go | 14 +++++--------- cmd/root/root.go | 2 +- codecov.yml | 1 - pkg/storages/e2e_test.go | 4 ++-- pkg/tools/ingest/loader.go | 12 +++++------- pkg/tools/ingest/sbom_test.go | 5 ++--- pkg/tools/ingest/vuln_test.go | 4 ++-- 11 files changed, 30 insertions(+), 47 deletions(-) diff --git a/api/v1/service_test.go b/api/v1/service_test.go index 111ed14..256363f 100644 --- a/api/v1/service_test.go +++ b/api/v1/service_test.go @@ -73,7 +73,7 @@ func TestGetNodesByGlob(t *testing.T) { func TestQueriesIngestAndCache(t *testing.T) { s := setupService() - result, err := ingest.LoadDataFromPath(s.storage, "../../testdata/osv-sboms/google_agi.sbom.json") + result, err := ingest.LoadDataFromPath("../../testdata/osv-sboms/google_agi.sbom.json") require.NoError(t, err) for _, data := range result { sbomReq := connect.NewRequest(&service.IngestSBOMRequest{ diff --git a/cmd/ingest/ingest.go b/cmd/ingest/ingest.go index 859e0f6..981a184 100644 --- a/cmd/ingest/ingest.go +++ b/cmd/ingest/ingest.go @@ -4,7 +4,6 @@ import ( "github.com/bitbomdev/minefield/cmd/ingest/osv" "github.com/bitbomdev/minefield/cmd/ingest/sbom" "github.com/bitbomdev/minefield/cmd/ingest/scorecard" - "github.com/bitbomdev/minefield/pkg/graph" "github.com/spf13/cobra" ) @@ -13,7 +12,7 @@ type options struct{} func (o *options) AddFlags(_ *cobra.Command) { } -func New(storage graph.Storage) *cobra.Command { +func New() *cobra.Command { cmd := &cobra.Command{ Use: "ingest", Short: "ingest metadata into the graph", @@ -21,8 +20,8 @@ func New(storage graph.Storage) *cobra.Command { DisableAutoGenTag: true, } - cmd.AddCommand(osv.New(storage)) - cmd.AddCommand(sbom.New(storage)) - cmd.AddCommand(scorecard.New(storage)) + cmd.AddCommand(osv.New()) + cmd.AddCommand(sbom.New()) + cmd.AddCommand(scorecard.New()) return cmd } diff --git a/cmd/ingest/osv/osv.go b/cmd/ingest/osv/osv.go index 9583a7a..28546b4 100644 --- a/cmd/ingest/osv/osv.go +++ b/cmd/ingest/osv/osv.go @@ -8,15 +8,13 @@ import ( "connectrpc.com/connect" apiv1 "github.com/bitbomdev/minefield/gen/api/v1" "github.com/bitbomdev/minefield/gen/api/v1/apiv1connect" - "github.com/bitbomdev/minefield/pkg/graph" "github.com/bitbomdev/minefield/pkg/tools" "github.com/bitbomdev/minefield/pkg/tools/ingest" "github.com/spf13/cobra" ) type options struct { - storage graph.Storage - addr string // Address of the minefield server + addr string // Address of the minefield server ingestServiceClient apiv1connect.IngestServiceClient } @@ -37,7 +35,7 @@ func (o *options) Run(_ *cobra.Command, args []string) error { } vulnsPath := args[0] // Ingest vulnerabilities - result, err := ingest.LoadDataFromPath(o.storage, vulnsPath) + result, err := ingest.LoadDataFromPath(vulnsPath) if err != nil { return fmt.Errorf("failed to load vulnerabilities: %w", err) } @@ -56,10 +54,8 @@ func (o *options) Run(_ *cobra.Command, args []string) error { return nil } -func New(storage graph.Storage) *cobra.Command { - o := &options{ - storage: storage, - } +func New() *cobra.Command { + o := &options{} cmd := &cobra.Command{ Use: "osv [path to vulnerability file/dir]", Short: "Graph vulnerability data into the graph, and connect it to existing library nodes", diff --git a/cmd/ingest/sbom/sbom.go b/cmd/ingest/sbom/sbom.go index 35cb361..db0c6ab 100644 --- a/cmd/ingest/sbom/sbom.go +++ b/cmd/ingest/sbom/sbom.go @@ -8,15 +8,13 @@ import ( "connectrpc.com/connect" apiv1 "github.com/bitbomdev/minefield/gen/api/v1" "github.com/bitbomdev/minefield/gen/api/v1/apiv1connect" - "github.com/bitbomdev/minefield/pkg/graph" "github.com/bitbomdev/minefield/pkg/tools" "github.com/bitbomdev/minefield/pkg/tools/ingest" "github.com/spf13/cobra" ) type options struct { - storage graph.Storage - addr string // Address of the minefield server + addr string // Address of the minefield server ingestServiceClient apiv1connect.IngestServiceClient } @@ -38,7 +36,7 @@ func (o *options) Run(_ *cobra.Command, args []string) error { } sbomPath := args[0] // Ingest SBOM - result, err := ingest.LoadDataFromPath(o.storage, sbomPath) + result, err := ingest.LoadDataFromPath(sbomPath) if err != nil { return fmt.Errorf("failed to ingest SBOM: %w", err) } @@ -59,10 +57,8 @@ func (o *options) Run(_ *cobra.Command, args []string) error { return nil } -func New(storage graph.Storage) *cobra.Command { - o := &options{ - storage: storage, - } +func New() *cobra.Command { + o := &options{} cmd := &cobra.Command{ Use: "sbom [path to sbom file/dir]", Short: "Ingest an sbom into the graph ", diff --git a/cmd/ingest/scorecard/scorecard.go b/cmd/ingest/scorecard/scorecard.go index e417723..4bee1e8 100644 --- a/cmd/ingest/scorecard/scorecard.go +++ b/cmd/ingest/scorecard/scorecard.go @@ -6,17 +6,15 @@ import ( "net/http" "connectrpc.com/connect" - "github.com/bitbomdev/minefield/gen/api/v1" + apiv1 "github.com/bitbomdev/minefield/gen/api/v1" "github.com/bitbomdev/minefield/gen/api/v1/apiv1connect" - "github.com/bitbomdev/minefield/pkg/graph" "github.com/bitbomdev/minefield/pkg/tools" "github.com/bitbomdev/minefield/pkg/tools/ingest" "github.com/spf13/cobra" ) type options struct { - storage graph.Storage - addr string // Address of the minefield server + addr string // Address of the minefield server ingestServiceClient apiv1connect.IngestServiceClient } @@ -38,7 +36,7 @@ func (o *options) Run(_ *cobra.Command, args []string) error { } scorecardPath := args[0] - result, err := ingest.LoadDataFromPath(o.storage, scorecardPath) + result, err := ingest.LoadDataFromPath(scorecardPath) if err != nil { return fmt.Errorf("failed to ingest SBOM: %w", err) } @@ -59,10 +57,8 @@ func (o *options) Run(_ *cobra.Command, args []string) error { return nil } -func New(storage graph.Storage) *cobra.Command { - o := &options{ - storage: storage, - } +func New() *cobra.Command { + o := &options{} cmd := &cobra.Command{ Use: "scorecard [path to scorecard file/dir]", Short: "Graph scorecard data into the graph, and connect it to existing library nodes", diff --git a/cmd/root/root.go b/cmd/root/root.go index cfdd841..ccb3f0f 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -32,7 +32,7 @@ func New(storage graph.Storage) *cobra.Command { o.AddFlags(cmd) cmd.AddCommand(query.New()) - cmd.AddCommand(ingest.New(storage)) + cmd.AddCommand(ingest.New()) cmd.AddCommand(cache.New()) cmd.AddCommand(leaderboard.New()) cmd.AddCommand(server.New(storage)) diff --git a/codecov.yml b/codecov.yml index 3ef1de7..ca00b81 100644 --- a/codecov.yml +++ b/codecov.yml @@ -10,6 +10,5 @@ coverage: threshold: 1% ignore: - - "cmd/**/*" - "gen/**/*" # generated code - "pkg/graph/mockGraph.go" # mockGraph is not covered diff --git a/pkg/storages/e2e_test.go b/pkg/storages/e2e_test.go index cef3d65..3e3599e 100644 --- a/pkg/storages/e2e_test.go +++ b/pkg/storages/e2e_test.go @@ -20,14 +20,14 @@ func TestParseAndExecute_E2E(t *testing.T) { vulnsPath := filepath.Join("..", "..", "testdata", "osv-vulns") // Ingest data from the folder - result, err := ingest.LoadDataFromPath(redisStorage, sbomPath) + result, err := ingest.LoadDataFromPath(sbomPath) assert.NoError(t, err) for _, data := range result { if err := ingest.SBOM(redisStorage, data.Data); err != nil { t.Fatalf("Failed to load SBOM from data: %v", err) } } - result, err = ingest.LoadDataFromPath(redisStorage, vulnsPath) + result, err = ingest.LoadDataFromPath(vulnsPath) assert.NoError(t, err) for _, data := range result { if err := ingest.Vulnerabilities(redisStorage, data.Data); err != nil { diff --git a/pkg/tools/ingest/loader.go b/pkg/tools/ingest/loader.go index 611e56e..97012a7 100644 --- a/pkg/tools/ingest/loader.go +++ b/pkg/tools/ingest/loader.go @@ -6,8 +6,6 @@ import ( "io" "os" "path/filepath" - - "github.com/bitbomdev/minefield/pkg/graph" ) // LoadDataFromPath takes in a directory or file path and processes the data into the storage. @@ -18,7 +16,7 @@ type Data struct { Data []byte } -func LoadDataFromPath(storage graph.Storage, path string) ([]Data, error) { +func LoadDataFromPath(path string) ([]Data, error) { info, err := os.Stat(path) if err != nil { return nil, fmt.Errorf("error accessing path %s: %w", path, err) @@ -35,7 +33,7 @@ func LoadDataFromPath(storage graph.Storage, path string) ([]Data, error) { } for _, entry := range entries { entryPath := filepath.Join(path, entry.Name()) - subResult, err := LoadDataFromPath(storage, entryPath) + subResult, err := LoadDataFromPath(entryPath) if err != nil { errors = append(errors, fmt.Errorf("failed to load data from path %s: %w", entryPath, err)) } else { @@ -48,7 +46,7 @@ func LoadDataFromPath(storage graph.Storage, path string) ([]Data, error) { } else { switch filepath.Ext(path) { case ".zip": - subResult, err := processZipFile(storage, path) + subResult, err := processZipFile(path) if err != nil { errors = append(errors, fmt.Errorf("failed to process zip file %s: %w", path, err)) } else { @@ -70,7 +68,7 @@ func LoadDataFromPath(storage graph.Storage, path string) ([]Data, error) { return result, nil } -func processZipFile(storage graph.Storage, filePath string) ([]Data, error) { +func processZipFile(filePath string) ([]Data, error) { r, err := zip.OpenReader(filePath) if err != nil { return nil, fmt.Errorf("failed to open zip file %s: %w", filePath, err) @@ -102,5 +100,5 @@ func processZipFile(storage graph.Storage, filePath string) ([]Data, error) { } } - return LoadDataFromPath(storage, tempDir) + return LoadDataFromPath(tempDir) } diff --git a/pkg/tools/ingest/sbom_test.go b/pkg/tools/ingest/sbom_test.go index d22464b..99bbffb 100644 --- a/pkg/tools/ingest/sbom_test.go +++ b/pkg/tools/ingest/sbom_test.go @@ -1,7 +1,6 @@ package ingest import ( - "os" "sort" "testing" @@ -37,7 +36,7 @@ func TestIngestSBOM(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { storage := graph.NewMockStorage() - result, err := LoadDataFromPath(storage, test.sbomPath) + result, err := LoadDataFromPath(test.sbomPath) if test.wantErr != (err != nil) { t.Errorf("Sbom() error = %v, wantErr = %v", err, test.wantErr) } @@ -46,7 +45,7 @@ func TestIngestSBOM(t *testing.T) { if err := SBOM(storage, data.Data); err != nil { if test.wantErr != (err != nil) { t.Errorf("Sbom() error = %v, wantErr = %v", err, test.wantErr) - } + } } } diff --git a/pkg/tools/ingest/vuln_test.go b/pkg/tools/ingest/vuln_test.go index 03040df..bb1089c 100644 --- a/pkg/tools/ingest/vuln_test.go +++ b/pkg/tools/ingest/vuln_test.go @@ -12,7 +12,7 @@ func TestVulnerabilities(t *testing.T) { vulnsDir := "../../../testdata/osv-vulns" sbomDir := "../../../testdata/osv-sboms" - result, err := LoadDataFromPath(storage, sbomDir) + result, err := LoadDataFromPath(sbomDir) if err != nil { t.Fatalf("Failed to ingest SBOM: %v", err) } @@ -33,7 +33,7 @@ func TestVulnerabilities(t *testing.T) { numberOfNodes := len(keys) - result, err = LoadDataFromPath(storage, vulnsDir) + result, err = LoadDataFromPath(vulnsDir) if err != nil { t.Fatalf("Failed to load vulnerabilities from directory %s: %v", vulnsDir, err) } From 21f662295e6f4c867b8280787d6ee8f544cc604f Mon Sep 17 00:00:00 2001 From: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> Date: Thu, 14 Nov 2024 19:17:49 -0600 Subject: [PATCH 3/7] Refactor codebase to remove `go.uber.org/fx` dependency and update storage initialization** - Remove dependency on `go.uber.org/fx` and associated dependency injection setup. - Simplify `main.go` by removing `fx.App` and directly executing the root command. - Modify `cmd/root/root.go`: - Rename `options` struct to `Options` to make it exportable. - Update `AddFlags` method to use exported fields. - Add `PersistentPreRun` function to start the pprof server if enabled. - Change `New` function signature to return `(*cobra.Command, error)` instead of just `*cobra.Command`. - Modify `cmd/server/server.go`: - Add `StorageType` and `StorageAddr` fields to options. - Implement `ProvideStorage` method to initialize storage based on `StorageType`. - Update `Run` method to initialize storage via `ProvideStorage`. - Remove `storage` parameter from the `New` function. - Adjust `New` function accordingly. - Update `go.mod` and `go.sum` to remove `fx` and related dependencies. - Delete unused `storages` module and its tests. - Move storage initialization logic into `cmd/server/server.go`. This refactoring removes the usage of the `go.uber.org/fx` dependency injection framework, simplifying the application startup process. Storage initialization is now handled within the server command and can be configured via command-line flags. This change streamlines the application and reduces unnecessary complexity. Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> --- .github/workflows/build.yaml | 4 +++ Makefile | 7 ++-- cmd/root/root.go | 47 ++++++++++++++----------- cmd/server/server.go | 54 ++++++++++++++++++++++++----- cmd/server/server_test.go | 2 +- cmd/server/wire.go | 24 +++++++++++++ cmd/server/wire_gen.go | 34 +++++++++++++++++++ go.mod | 5 +-- go.sum | 64 +++++++++++++++++++++++++++++------ main.go | 42 +++++------------------ pkg/storages/fxmodule.go | 14 -------- pkg/storages/fxmodule_test.go | 46 ------------------------- pkg/storages/redis_storage.go | 7 ++-- 13 files changed, 210 insertions(+), 140 deletions(-) create mode 100644 cmd/server/wire.go create mode 100644 cmd/server/wire_gen.go delete mode 100644 pkg/storages/fxmodule.go delete mode 100644 pkg/storages/fxmodule_test.go diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bffb81a..89ca4cc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -18,6 +18,10 @@ jobs: uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v2 with: go-version: '1.22.5' + - name: Install Wire + run: | + go install github.com/google/wire/cmd/wire@latest + echo "$HOME/go/bin" >> $GITHUB_PATH - name: Build run: make build && make git-porcelain diff --git a/Makefile b/Makefile index 6d25f49..ac2a993 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ .DEFAULT_GOAL := all # Build target -build: +build: wire go build -o bin/minefield main.go # Test target @@ -40,6 +40,9 @@ go-mod-tidy: git-porcelain: git status --porcelain -all: build test docker-build go-mod-tidy git-porcelain +wire: + cd cmd/server && wire + +all: build test docker-build go-mod-tidy git-porcelain wire .PHONY: test test-e2e build clean clean-redis docker-up docker-down docker-logs docker-build all buf-generate install-buf diff --git a/cmd/root/root.go b/cmd/root/root.go index ccb3f0f..35591aa 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -1,41 +1,50 @@ package root import ( + "fmt" + "net/http" + "github.com/bitbomdev/minefield/cmd/cache" "github.com/bitbomdev/minefield/cmd/ingest" "github.com/bitbomdev/minefield/cmd/leaderboard" "github.com/bitbomdev/minefield/cmd/query" "github.com/bitbomdev/minefield/cmd/server" - "github.com/bitbomdev/minefield/pkg/graph" "github.com/spf13/cobra" ) -type options struct { - pprofAddr string - pprofEnabled bool +type Options struct { + PprofAddr string + PprofEnabled bool } -func (o *options) AddFlags(cmd *cobra.Command) { - cmd.PersistentFlags().BoolVar(&o.pprofEnabled, "pprof", false, "enable pprof server") - cmd.PersistentFlags().StringVar(&o.pprofAddr, "pprof-addr", "localhost:6060", "address for pprof server") +func (o *Options) AddFlags(cmd *cobra.Command) { + cmd.PersistentFlags().BoolVar(&o.PprofEnabled, "pprof", false, "Enable pprof server") + cmd.PersistentFlags().StringVar(&o.PprofAddr, "pprof-addr", "localhost:6060", "Address for pprof server") } -func New(storage graph.Storage) *cobra.Command { - o := &options{} - cmd := &cobra.Command{ +func New() (*cobra.Command, error) { + o := &Options{} + rootCmd := &cobra.Command{ Use: "minefield", - Short: "graphing SBOM's with the power of roaring bitmaps", + Short: "Graphing SBOM's with the power of roaring bitmaps", SilenceUsage: true, DisableAutoGenTag: true, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + o.AddFlags(cmd) + if o.PprofEnabled { + go func() { + fmt.Printf("Starting pprof server on %s\n", o.PprofAddr) + http.ListenAndServe(o.PprofAddr, nil) + }() + } + }, } - o.AddFlags(cmd) - - cmd.AddCommand(query.New()) - cmd.AddCommand(ingest.New()) - cmd.AddCommand(cache.New()) - cmd.AddCommand(leaderboard.New()) - cmd.AddCommand(server.New(storage)) + rootCmd.AddCommand(query.New()) + rootCmd.AddCommand(ingest.New()) + rootCmd.AddCommand(cache.New()) + rootCmd.AddCommand(leaderboard.New()) + rootCmd.AddCommand(server.New()) - return cmd + return rootCmd, nil } diff --git a/cmd/server/server.go b/cmd/server/server.go index d438720..8d98597 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -14,6 +14,7 @@ import ( service "github.com/bitbomdev/minefield/api/v1" "github.com/bitbomdev/minefield/gen/api/v1/apiv1connect" "github.com/bitbomdev/minefield/pkg/graph" + "github.com/bitbomdev/minefield/pkg/storages" "github.com/spf13/cobra" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" @@ -23,17 +24,42 @@ type options struct { storage graph.Storage concurrency int32 addr string + StorageType string + StorageAddr string } +const ( + defaultConcurrency = 10 + defaultAddr = "localhost:8089" + redisStorageType = "redis" +) + func (o *options) AddFlags(cmd *cobra.Command) { - cmd.Flags().Int32Var(&o.concurrency, "concurrency", 10, "Maximum number of concurrent operations for leaderboard operations") - cmd.Flags().StringVar(&o.addr, "addr", "localhost:8089", "Network address and port for the server (e.g. localhost:8089)") + cmd.Flags().Int32Var(&o.concurrency, "concurrency", defaultConcurrency, "Maximum number of concurrent operations for leaderboard operations") + cmd.Flags().StringVar(&o.addr, "addr", defaultAddr, "Network address and port for the server (e.g. localhost:8089)") + cmd.Flags().StringVar(&o.StorageType, "storage-type", redisStorageType, "Type of storage to use (e.g., redis, sql)") + cmd.Flags().StringVar(&o.StorageAddr, "storage-addr", "localhost:6379", "Address for storage backend") +} + +func (o *options) ProvideStorage() (graph.Storage, error) { + switch o.StorageType { + case redisStorageType: + return storages.NewRedisStorage(o.StorageAddr) + default: + return nil, fmt.Errorf("unknown storage type: %s", o.StorageType) + } } func (o *options) Run(cmd *cobra.Command, args []string) error { + var err error + o.storage, err = o.ProvideStorage() + if err != nil { + return fmt.Errorf("failed to initialize storage: %w", err) + } + server, err := o.setupServer() if err != nil { - return err + return fmt.Errorf("failed to setup server: %w", err) } return o.startServer(server) } @@ -45,7 +71,7 @@ func (o *options) setupServer() (*http.Server, error) { serviceAddr := o.addr if serviceAddr == "" { - serviceAddr = "localhost:8089" + serviceAddr = defaultAddr } newService := service.NewService(o.storage, o.concurrency) @@ -98,10 +124,8 @@ func (o *options) startServer(server *http.Server) error { } // New returns a new cobra command for the server. -func New(storage graph.Storage) *cobra.Command { - o := &options{ - storage: storage, - } +func New() *cobra.Command { + o := &options{} cmd := &cobra.Command{ Use: "server", Short: "Start the minefield server for graph operations and queries", @@ -110,6 +134,18 @@ func New(storage graph.Storage) *cobra.Command { DisableAutoGenTag: true, } o.AddFlags(cmd) - return cmd } + +func NewServerCommand(storage graph.Storage, o *options) (*cobra.Command, error) { + o.storage = storage + cmd := &cobra.Command{ + Use: "server", + Short: "Start the minefield server for graph operations and queries", + Args: cobra.ExactArgs(0), + RunE: o.Run, + DisableAutoGenTag: true, + } + o.AddFlags(cmd) + return cmd, nil +} diff --git a/cmd/server/server_test.go b/cmd/server/server_test.go index b2166bd..13c5550 100644 --- a/cmd/server/server_test.go +++ b/cmd/server/server_test.go @@ -90,7 +90,7 @@ func TestNew(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cmd := New(tt.storage) + cmd := New() assert.NotNil(t, cmd) assert.Equal(t, tt.want.use, cmd.Use) diff --git a/cmd/server/wire.go b/cmd/server/wire.go new file mode 100644 index 0000000..ad666cf --- /dev/null +++ b/cmd/server/wire.go @@ -0,0 +1,24 @@ +//go:build wireinject +// +build wireinject + +package server + +import ( + "github.com/bitbomdev/minefield/pkg/graph" + "github.com/google/wire" + "github.com/spf13/cobra" +) + +const redis = "redis" + +func InitializeServerCommand(o *options) (*cobra.Command, error) { + wire.Build( + ProvideStorage, + NewServerCommand, + ) + return nil, nil +} + +func ProvideStorage(o *options) (graph.Storage, error) { + return o.ProvideStorage() +} diff --git a/cmd/server/wire_gen.go b/cmd/server/wire_gen.go new file mode 100644 index 0000000..f6835ac --- /dev/null +++ b/cmd/server/wire_gen.go @@ -0,0 +1,34 @@ +// Code generated by Wire. DO NOT EDIT. + +//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:build !wireinject +// +build !wireinject + +package server + +import ( + "github.com/bitbomdev/minefield/pkg/graph" + "github.com/spf13/cobra" +) + +// Injectors from wire.go: + +func InitializeServerCommand(o *options) (*cobra.Command, error) { + storage, err := ProvideStorage(o) + if err != nil { + return nil, err + } + command, err := NewServerCommand(storage, o) + if err != nil { + return nil, err + } + return command, nil +} + +// wire.go: + +const redis = "redis" + +func ProvideStorage(o *options) (graph.Storage, error) { + return o.ProvideStorage() +} diff --git a/go.mod b/go.mod index 6892081..8e77b4f 100644 --- a/go.mod +++ b/go.mod @@ -11,13 +11,13 @@ require ( github.com/go-redis/redis/v8 v8.11.5 github.com/goccy/go-json v0.10.3 github.com/google/go-cmp v0.6.0 + github.com/google/wire v0.6.0 github.com/olekukonko/tablewriter v0.0.5 github.com/package-url/packageurl-go v0.1.3 github.com/protobom/protobom v0.5.0 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 github.com/zeebo/assert v1.3.1 - go.uber.org/fx v1.23.0 golang.org/x/net v0.31.0 google.golang.org/protobuf v1.35.2 ) @@ -46,9 +46,6 @@ require ( github.com/spdx/tools-golang v0.5.5 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect - go.uber.org/dig v1.18.0 // indirect - go.uber.org/multierr v1.10.0 // indirect - go.uber.org/zap v1.26.0 // indirect golang.org/x/sys v0.27.0 // indirect golang.org/x/text v0.20.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/go.sum b/go.sum index 495c8c2..196317c 100644 --- a/go.sum +++ b/go.sum @@ -38,11 +38,15 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/wire v0.6.0 h1:HBkoIh4BdSxoyo9PveV8giw7ZsaBOvzWKfcg/6MrVwI= +github.com/google/wire v0.6.0/go.mod h1:F4QhpQ9EDIdJ1Mbop/NZBRB+5yrR6qg3BnctaoUk6NA= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -103,25 +107,65 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.1 h1:vukIABvugfNMZMQO1ABsyQDJDTVQbn+LWSMy1ol1h6A= github.com/zeebo/assert v1.3.1/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -go.uber.org/dig v1.18.0 h1:imUL1UiY0Mg4bqbFfsRQO5G4CGRBec/ZujWTvSVp3pw= -go.uber.org/dig v1.18.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= -go.uber.org/fx v1.23.0 h1:lIr/gYWQGfTwGcSXWXu4vP5Ws6iqnNEIY+F/aFzCKTg= -go.uber.org/fx v1.23.0/go.mod h1:o/D9n+2mLP6v1EG+qsdT1O8wKopYAsqZasju97SDFCU= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go index 38fa19d..7c93856 100644 --- a/main.go +++ b/main.go @@ -2,43 +2,19 @@ package main import ( "fmt" - "net/http" - _ "net/http/pprof" // Import for side-effect + "os" "github.com/bitbomdev/minefield/cmd/root" - "github.com/bitbomdev/minefield/pkg/graph" - "github.com/bitbomdev/minefield/pkg/storages" - "github.com/spf13/cobra" - "go.uber.org/fx" ) func main() { - var pprofEnabled bool - var pprofAddr string + rootCmd, err := root.New() + if err != nil { + fmt.Fprintf(os.Stderr, "Error initializing root command: %v\n", err) + os.Exit(1) + } - app := fx.New( - storages.NewRedisStorageModule("localhost:6379"), - fx.Invoke(func(storage graph.Storage, shutdowner fx.Shutdowner) { - rootCmd := root.New(storage) - rootCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { - pprofEnabled, _ = cmd.Flags().GetBool("pprof") - pprofAddr, _ = cmd.Flags().GetString("pprof-addr") - if pprofEnabled { - go func() { - fmt.Printf("Starting pprof server on %s\n", pprofAddr) - http.ListenAndServe(pprofAddr, nil) - }() - } - } - if err := rootCmd.Execute(); err != nil { - panic(err) - } - if err := shutdowner.Shutdown(); err != nil { - panic(fmt.Sprintf("Failed to shutdown fx err = %s", err)) - } - }), - fx.NopLogger, - ) - - app.Run() + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } } diff --git a/pkg/storages/fxmodule.go b/pkg/storages/fxmodule.go deleted file mode 100644 index 9dab0cf..0000000 --- a/pkg/storages/fxmodule.go +++ /dev/null @@ -1,14 +0,0 @@ -package storages - -import ( - "github.com/bitbomdev/minefield/pkg/graph" - "go.uber.org/fx" -) - -func NewRedisStorageModule(addr string) fx.Option { - return fx.Provide( - func() graph.Storage { - return NewRedisStorage(addr) - }, - ) -} diff --git a/pkg/storages/fxmodule_test.go b/pkg/storages/fxmodule_test.go deleted file mode 100644 index 1b35bd8..0000000 --- a/pkg/storages/fxmodule_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package storages - -import ( - "context" - "testing" - - "github.com/bitbomdev/minefield/pkg/graph" - "github.com/go-redis/redis/v8" - "github.com/stretchr/testify/assert" - "go.uber.org/fx" -) - -func TestNewRedisStorageModule(t *testing.T) { - setupTestRedis() - // Base address for Redis - var addr string - var client *redis.Client - var err error - - // Use a static port for Redis - addr = "localhost:6379" - client = redis.NewClient(&redis.Options{Addr: addr}) - defer client.Close() - - _, err = client.Ping(context.Background()).Result() - if err != nil { - t.Skipf("Skipping testdata: Redis is not available at %s", addr) - } - - // Create an fx.App for testing - app := fx.New( - NewRedisStorageModule(addr), - fx.Invoke(func(storage graph.Storage) { - assert.Implements(t, (*graph.Storage)(nil), storage, "RedisStorage should implement graph.Storage") - - // Perform additional assertions on the storage instance - redisStorage, ok := storage.(*RedisStorage) - assert.True(t, ok, "Expected storage to be of type *RedisStorage") - assert.NotNil(t, redisStorage.Client, "RedisStorage Client should not be nil") - }), - ) - - // Start and stop the fx.App - assert.NoError(t, app.Start(context.Background())) - assert.NoError(t, app.Stop(context.Background())) -} diff --git a/pkg/storages/redis_storage.go b/pkg/storages/redis_storage.go index f1e2f09..851f55e 100644 --- a/pkg/storages/redis_storage.go +++ b/pkg/storages/redis_storage.go @@ -14,11 +14,14 @@ type RedisStorage struct { Client *redis.Client } -func NewRedisStorage(addr string) graph.Storage { +func NewRedisStorage(addr string) (graph.Storage, error) { rdb := redis.NewClient(&redis.Options{ Addr: addr, }) - return &RedisStorage{Client: rdb} + if err := rdb.Ping(context.Background()).Err(); err != nil { + return nil, fmt.Errorf("failed to connect to Redis: %w", err) + } + return &RedisStorage{Client: rdb}, nil } func (r *RedisStorage) GenerateID() (uint32, error) { From 2a6ab15ff54492b7946b2193079cdc014c9bf1df Mon Sep 17 00:00:00 2001 From: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> Date: Fri, 15 Nov 2024 08:22:33 -0600 Subject: [PATCH 4/7] Some code clean up Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> --- .github/workflows/build.yaml | 4 +-- .github/workflows/grype.yaml | 31 +++++++--------- Makefile | 13 +++---- cmd/ingest/ingest_test.go | 33 +++++++++++++++++ cmd/ingest/osv/osv_test.go | 50 ++++++++++++++++++++++++++ cmd/ingest/sbom/sbom_test.go | 29 +++++++++++++++ cmd/ingest/scorecard/scorecard_test.go | 29 +++++++++++++++ cmd/root/root.go | 11 +++++- cmd/server/wire.go | 2 -- cmd/server/wire_gen.go | 2 -- go.mod | 2 +- 11 files changed, 172 insertions(+), 34 deletions(-) create mode 100644 cmd/ingest/ingest_test.go create mode 100644 cmd/ingest/osv/osv_test.go create mode 100644 cmd/ingest/sbom/sbom_test.go create mode 100644 cmd/ingest/scorecard/scorecard_test.go diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 89ca4cc..fe52c59 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -17,11 +17,11 @@ jobs: - name: Set up Go uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v2 with: - go-version: '1.22.5' + go-version: '1.23' - name: Install Wire run: | go install github.com/google/wire/cmd/wire@latest - echo "$HOME/go/bin" >> $GITHUB_PATH + echo "$HOME/go/bin" >> "$GITHUB_PATH" - name: Build run: make build && make git-porcelain diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml index b113ec0..0ecd2b9 100644 --- a/.github/workflows/grype.yaml +++ b/.github/workflows/grype.yaml @@ -3,32 +3,28 @@ name: Docker Image Scan on: pull_request: branches: [ main ] - paths: - - 'Dockerfile' - - 'docker-compose*.yml' - - '.docker/**' jobs: scan: name: Build and Scan runs-on: ubuntu-latest - environment: security concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true steps: - - name: Notify on failure - if: failure() - uses: actions/github-script@v6 - with: - script: | - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: '❌ Container security scan failed. Please check the workflow logs.' - }) + - name: Notify on failure + if: failure() + uses: actions/github-script@v6 + with: + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '❌ Container security scan failed. Please check the workflow logs.' + }) + - name: Checkout code uses: actions/checkout@v3 @@ -53,5 +49,4 @@ jobs: uses: github/codeql-action/upload-sarif@v2 if: success() || failure() with: - sarif_file: results.sarif - + sarif_file: results.sarif \ No newline at end of file diff --git a/Makefile b/Makefile index ac2a993..30bda6d 100644 --- a/Makefile +++ b/Makefile @@ -1,27 +1,21 @@ -# Default target .DEFAULT_GOAL := all -# Build target build: wire go build -o bin/minefield main.go -# Test target test: go test -v -coverprofile=coverage.out ./... test-e2e: docker-up e2e=true go test -v -coverprofile=coverage.out ./... -# Clean target clean: rm -rf bin -# Clean Redis data clean-redis: docker compose exec -T redis redis-cli ping || docker compose up -d redis docker compose exec -T redis redis-cli FLUSHALL -# Docker targets docker-up: docker-down docker compose up -d @@ -40,9 +34,12 @@ go-mod-tidy: git-porcelain: git status --porcelain -wire: +check-wire: + @command -v wire >/dev/null 2>&1 || { echo >&2 "wire is not installed. Please install wire. go install github.com/google/wire/cmd/wire@latest"; exit 1; } + +wire: check-wire cd cmd/server && wire all: build test docker-build go-mod-tidy git-porcelain wire -.PHONY: test test-e2e build clean clean-redis docker-up docker-down docker-logs docker-build all buf-generate install-buf +.PHONY: test test-e2e build clean clean-redis docker-up docker-down docker-logs docker-build all wire \ No newline at end of file diff --git a/cmd/ingest/ingest_test.go b/cmd/ingest/ingest_test.go new file mode 100644 index 0000000..c6c8633 --- /dev/null +++ b/cmd/ingest/ingest_test.go @@ -0,0 +1,33 @@ +package ingest + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewCommand(t *testing.T) { + cmd := New() + assert.NotNil(t, cmd, "Ingest command should not be nil") + + assert.Equal(t, "ingest", cmd.Use, "Command 'Use' should be 'ingest'") + assert.Equal(t, "ingest metadata into the graph", cmd.Short, "Command 'Short' description should match") + + subcommands := cmd.Commands() + subcommandUses := []string{} + for _, subcmd := range subcommands { + subcommandUses = append(subcommandUses, subcmd.Use) + } + + expectedSubcommands := []string{ + "osv [path to vulnerability file/dir]", + "sbom [path to sbom file/dir]", + "scorecard [path to scorecard file/dir]", + } + assert.ElementsMatch(t, expectedSubcommands, subcommandUses, "Subcommands should match expected list") +} + +func TestWireDependencyResolution(t *testing.T) { + cmd := New() + assert.NotNil(t, cmd, "Ingest command should initialize without error") +} diff --git a/cmd/ingest/osv/osv_test.go b/cmd/ingest/osv/osv_test.go new file mode 100644 index 0000000..cdb56d9 --- /dev/null +++ b/cmd/ingest/osv/osv_test.go @@ -0,0 +1,50 @@ +package osv + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/spf13/pflag" +) + +func TestNew(t *testing.T) { + tests := []struct { + name string + wantUse string + wantShort string + wantFlagCount int + }{ + { + name: "creates command with correct configuration", + wantUse: "osv [path to vulnerability file/dir]", + wantShort: "Graph vulnerability data into the graph, and connect it to existing library nodes", + wantFlagCount: 1, // Should have the "addr" flag + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmd := New() + + // Test basic command properties + assert.Equal(t, tt.wantUse, cmd.Use) + assert.Equal(t, tt.wantShort, cmd.Short) + assert.True(t, cmd.DisableAutoGenTag) + assert.NotNil(t, cmd.RunE) + + // Test flags + flags := cmd.Flags() + + // Count the number of defined flags + flagCount := 0 + flags.VisitAll(func(*pflag.Flag) { flagCount++ }) + assert.Equal(t, tt.wantFlagCount, flagCount) + + // Test addr flag specifically + addrFlag := flags.Lookup("addr") + assert.NotNil(t, addrFlag) + assert.Equal(t, "string", addrFlag.Value.Type()) + assert.Equal(t, DefaultAddr, addrFlag.DefValue) + }) + } +} diff --git a/cmd/ingest/sbom/sbom_test.go b/cmd/ingest/sbom/sbom_test.go new file mode 100644 index 0000000..9ce05f1 --- /dev/null +++ b/cmd/ingest/sbom/sbom_test.go @@ -0,0 +1,29 @@ +package sbom + +import ( + "testing" +) + +func TestNew(t *testing.T) { + cmd := New() + + if cmd.Use != "sbom [path to sbom file/dir]" { + t.Errorf("expected Use to be 'sbom [path to sbom file/dir]', got %s", cmd.Use) + } + + if cmd.Short != "Ingest an sbom into the graph " { + t.Errorf("expected Short to be 'Ingest an sbom into the graph ', got %s", cmd.Short) + } + + if cmd.Args == nil || cmd.Args(nil, []string{"arg1"}) != nil { + t.Errorf("expected Args to be cobra.ExactArgs(1)") + } + + if cmd.DisableAutoGenTag != true { + t.Errorf("expected DisableAutoGenTag to be true") + } + + if cmd.RunE == nil { + t.Errorf("expected RunE to be set") + } +} diff --git a/cmd/ingest/scorecard/scorecard_test.go b/cmd/ingest/scorecard/scorecard_test.go new file mode 100644 index 0000000..ff5c3b9 --- /dev/null +++ b/cmd/ingest/scorecard/scorecard_test.go @@ -0,0 +1,29 @@ +package scorecard + +import ( + "testing" +) + +func TestNew(t *testing.T) { + cmd := New() + + if cmd.Use != "scorecard [path to scorecard file/dir]" { + t.Errorf("expected Use to be 'scorecard [path to scorecard file/dir]', got %s", cmd.Use) + } + + if cmd.Short != "Graph scorecard data into the graph, and connect it to existing library nodes" { + t.Errorf("expected Short to be 'Graph scorecard data into the graph, and connect it to existing library nodes', got %s", cmd.Short) + } + + if cmd.Args == nil || cmd.Args(nil, []string{"arg1"}) != nil { + t.Errorf("expected Args to be cobra.ExactArgs(1)") + } + + if cmd.DisableAutoGenTag != true { + t.Errorf("expected DisableAutoGenTag to be true") + } + + if cmd.RunE == nil { + t.Errorf("expected RunE to be set") + } +} diff --git a/cmd/root/root.go b/cmd/root/root.go index 35591aa..1d523b6 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -1,6 +1,7 @@ package root import ( + "context" "fmt" "net/http" @@ -32,10 +33,18 @@ func New() (*cobra.Command, error) { PersistentPreRun: func(cmd *cobra.Command, args []string) { o.AddFlags(cmd) if o.PprofEnabled { + srv := &http.Server{Addr: o.PprofAddr} go func() { fmt.Printf("Starting pprof server on %s\n", o.PprofAddr) - http.ListenAndServe(o.PprofAddr, nil) + if err := srv.ListenAndServe(); err != http.ErrServerClosed { + fmt.Printf("pprof server error: %v\n", err) + } }() + cmd.PersistentPostRun = func(cmd *cobra.Command, args []string) { + if err := srv.Shutdown(context.Background()); err != nil { + fmt.Printf("pprof server shutdown error: %v\n", err) + } + } } }, } diff --git a/cmd/server/wire.go b/cmd/server/wire.go index ad666cf..c11a973 100644 --- a/cmd/server/wire.go +++ b/cmd/server/wire.go @@ -9,8 +9,6 @@ import ( "github.com/spf13/cobra" ) -const redis = "redis" - func InitializeServerCommand(o *options) (*cobra.Command, error) { wire.Build( ProvideStorage, diff --git a/cmd/server/wire_gen.go b/cmd/server/wire_gen.go index f6835ac..b44c217 100644 --- a/cmd/server/wire_gen.go +++ b/cmd/server/wire_gen.go @@ -27,8 +27,6 @@ func InitializeServerCommand(o *options) (*cobra.Command, error) { // wire.go: -const redis = "redis" - func ProvideStorage(o *options) (graph.Storage, error) { return o.ProvideStorage() } diff --git a/go.mod b/go.mod index 8e77b4f..9f80691 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/package-url/packageurl-go v0.1.3 github.com/protobom/protobom v0.5.0 github.com/spf13/cobra v1.8.1 + github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 github.com/zeebo/assert v1.3.1 golang.org/x/net v0.31.0 @@ -44,7 +45,6 @@ require ( github.com/onsi/gomega v1.30.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spdx/tools-golang v0.5.5 // indirect - github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect golang.org/x/sys v0.27.0 // indirect golang.org/x/text v0.20.0 // indirect From c4c1928c0ba6d43c5a5b6c20feeb37c3e28f9655 Mon Sep 17 00:00:00 2001 From: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> Date: Fri, 15 Nov 2024 09:42:32 -0600 Subject: [PATCH 5/7] Fixed the codereview issues Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> --- Makefile | 2 +- cmd/root/root.go | 4 ++-- main.go | 7 +------ 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 30bda6d..1045966 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ check-wire: @command -v wire >/dev/null 2>&1 || { echo >&2 "wire is not installed. Please install wire. go install github.com/google/wire/cmd/wire@latest"; exit 1; } wire: check-wire - cd cmd/server && wire + cd cmd/server && wire || { echo "Wire generation failed in cmd/server"; exit 1; } all: build test docker-build go-mod-tidy git-porcelain wire diff --git a/cmd/root/root.go b/cmd/root/root.go index 1d523b6..ed74a59 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -23,7 +23,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) { cmd.PersistentFlags().StringVar(&o.PprofAddr, "pprof-addr", "localhost:6060", "Address for pprof server") } -func New() (*cobra.Command, error) { +func New() *cobra.Command { o := &Options{} rootCmd := &cobra.Command{ Use: "minefield", @@ -55,5 +55,5 @@ func New() (*cobra.Command, error) { rootCmd.AddCommand(leaderboard.New()) rootCmd.AddCommand(server.New()) - return rootCmd, nil + return rootCmd } diff --git a/main.go b/main.go index 7c93856..36e0f77 100644 --- a/main.go +++ b/main.go @@ -1,18 +1,13 @@ package main import ( - "fmt" "os" "github.com/bitbomdev/minefield/cmd/root" ) func main() { - rootCmd, err := root.New() - if err != nil { - fmt.Fprintf(os.Stderr, "Error initializing root command: %v\n", err) - os.Exit(1) - } + rootCmd := root.New() if err := rootCmd.Execute(); err != nil { os.Exit(1) From 7fa336defe6d89469921b3ee6762b75278bf6fcc Mon Sep 17 00:00:00 2001 From: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> Date: Fri, 15 Nov 2024 09:52:25 -0600 Subject: [PATCH 6/7] Cleaned up Signed-off-by: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> --- cmd/root/root.go | 6 +- go.mod | 1 - go.sum | 2 - pkg/graph/visualizer.go | 201 ------------------------------- pkg/graph/visualizer_test.go | 221 ----------------------------------- 5 files changed, 3 insertions(+), 428 deletions(-) delete mode 100644 pkg/graph/visualizer.go delete mode 100644 pkg/graph/visualizer_test.go diff --git a/cmd/root/root.go b/cmd/root/root.go index ed74a59..7261813 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -13,18 +13,18 @@ import ( "github.com/spf13/cobra" ) -type Options struct { +type options struct { PprofAddr string PprofEnabled bool } -func (o *Options) AddFlags(cmd *cobra.Command) { +func (o *options) AddFlags(cmd *cobra.Command) { cmd.PersistentFlags().BoolVar(&o.PprofEnabled, "pprof", false, "Enable pprof server") cmd.PersistentFlags().StringVar(&o.PprofAddr, "pprof-addr", "localhost:6060", "Address for pprof server") } func New() *cobra.Command { - o := &Options{} + o := &options{} rootCmd := &cobra.Command{ Use: "minefield", Short: "Graphing SBOM's with the power of roaring bitmaps", diff --git a/go.mod b/go.mod index 9f80691..d75ba73 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/Masterminds/semver v1.5.0 github.com/RoaringBitmap/roaring v1.9.4 github.com/alecthomas/participle/v2 v2.1.1 - github.com/go-echarts/go-echarts/v2 v2.4.5 github.com/go-redis/redis/v8 v8.11.5 github.com/goccy/go-json v0.10.3 github.com/google/go-cmp v0.6.0 diff --git a/go.sum b/go.sum index 196317c..ec64feb 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,6 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/go-echarts/go-echarts/v2 v2.4.5 h1:gwDqxdi5x329sg+g2ws2OklreJ1K34FCimraInurzwk= -github.com/go-echarts/go-echarts/v2 v2.4.5/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= diff --git a/pkg/graph/visualizer.go b/pkg/graph/visualizer.go deleted file mode 100644 index 5581f2a..0000000 --- a/pkg/graph/visualizer.go +++ /dev/null @@ -1,201 +0,0 @@ -package graph - -import ( - "context" - "errors" - "fmt" - "math" - "math/rand" - "net/http" - "time" - - "github.com/RoaringBitmap/roaring" - "github.com/go-echarts/go-echarts/v2/charts" - "github.com/go-echarts/go-echarts/v2/opts" -) - -func RunGraphVisualizer(storage Storage, ids *roaring.Bitmap, query string, server *http.Server) (func(), error) { - server.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - chart, err := graphQuery(storage, ids, query) - if err != nil { - http.Error(w, "Error generating graph: "+err.Error(), http.StatusInternalServerError) - return - } - err = chart.Render(w) - if err != nil { - http.Error(w, "Error rendering graph: "+err.Error(), http.StatusInternalServerError) - return - } - fmt.Println("Graph rendered successfully") - }) - - go func() { - if err := server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { - fmt.Printf("HTTP server ListenAndServe: %v", err) - } - }() - - fmt.Printf("Starting server on localhost%s\n", server.Addr) - - shutdown := func() { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := server.Shutdown(ctx); err != nil { - fmt.Printf("Server forced to shutdown: %v\n", err) - } - fmt.Println("Server stopped") - } - - return shutdown, nil -} - -func graphQuery(storage Storage, ids *roaring.Bitmap, query string) (*charts.Graph, error) { - graph := charts.NewGraph() - graph.SetGlobalOptions( - charts.WithTitleOpts(opts.Title{ - Title: query, - }), - charts.WithAnimation(false), - charts.WithInitializationOpts(opts.Initialization{ - Width: "9000px", - Height: "5000px", - // Theme: "dark", - }), - ) - - var nodes []opts.GraphNode - var links []opts.GraphLink - - alreadyCreatedNodes := roaring.New() - alreadyCreatedLinks := make(map[uint32]*roaring.Bitmap) - - for _, id := range ids.ToArray() { - node, err := storage.GetNode(id) - if err != nil { - return nil, err - } - connections := 0 - - for _, dep := range append(node.Children.ToArray(), node.Parents.ToArray()...) { - if ids.Contains(dep) { - connections++ - } - } - if !alreadyCreatedNodes.Contains(id) { - alreadyCreatedNodes.Add(id) - symbolSize := calculateSymbolSize(connections) - color := getColorForSize(symbolSize) - nodes = append(nodes, opts.GraphNode{ - SymbolSize: symbolSize, - Name: node.Name, - ItemStyle: &opts.ItemStyle{Color: color}, - X: float32(rand.Intn(100000)), - Y: float32(rand.Intn(100000)), - }) - } - - } - for _, id := range ids.ToArray() { - node, err := storage.GetNode(id) - if err != nil { - return nil, err - } - - for _, dep := range append(node.Children.ToArray(), node.Parents.ToArray()...) { - - depNode, err := storage.GetNode(dep) - if err != nil { - return nil, err - } - - if alreadyCreatedNodes.Contains(dep) { - if bitmap := alreadyCreatedLinks[id]; bitmap == nil || !bitmap.Contains(dep) { - links = append(links, opts.GraphLink{Source: node.Name, Target: depNode.Name}) - if bitmap == nil { - bitmap = roaring.New() - } - bitmap.Add(dep) - alreadyCreatedLinks[id] = bitmap - - if oppBitmap := alreadyCreatedLinks[dep]; oppBitmap == nil { - oppBitmap = roaring.New() - alreadyCreatedLinks[dep] = oppBitmap - } - alreadyCreatedLinks[dep].Add(id) - } - } - } - } - - graph.AddSeries("graph", nodes, links). - SetSeriesOptions( - charts.WithGraphChartOpts(opts.GraphChart{ - Roam: opts.Bool(true), - FocusNodeAdjacency: opts.Bool(true), - Force: &opts.GraphForce{Repulsion: 80000000, InitLayout: "circular", EdgeLength: 20}, - }), - charts.WithEmphasisOpts(opts.Emphasis{ - Label: &opts.Label{ - Show: opts.Bool(true), - Color: "white", - Position: "left", - }, - }), - charts.WithLineStyleOpts(opts.LineStyle{ - Curveness: 0.1, - }), - ) - fmt.Println("Graph generated successfully") - fmt.Printf("Number of nodes: %d\n", len(nodes)) - fmt.Printf("Number of links: %d\n", len(links)) - return graph, nil -} - -func getColorForSize(size int) string { - // Map size to a value between 0 and 1 - t := math.Max(0, math.Min(1, float64(size-10)/50)) // Clamp t between 0 and 1 - - // Define color stops (muted versions) - colors := []struct{ r, g, b uint8 }{ - {139, 0, 0}, // Dark red (smallest nodes) - {165, 42, 42}, // Brown - {178, 34, 34}, // Firebrick - {205, 92, 92}, // Indian red - {210, 105, 30}, // Chocolate - {205, 133, 63}, // Peru - {210, 105, 30}, // Muted orange (middle nodes) - {188, 143, 143}, // Rosy brown - {199, 21, 133}, // Medium violet red - {186, 85, 211}, // Medium orchid (replacing Pale violet red) - {255, 20, 147}, // Deep pink (largest nodes) - } - - // Find the two colors to interpolate between - i := int(t * float64(len(colors)-1)) - i = int(math.Min(float64(len(colors)-2), float64(i))) // Ensure i is within bounds - - c1, c2 := colors[i], colors[i+1] - - // Interpolate between the two colors - f := t*float64(len(colors)-1) - float64(i) - r := uint8(float64(c1.r)*(1-f) + float64(c2.r)*f) - g := uint8(float64(c1.g)*(1-f) + float64(c2.g)*f) - b := uint8(float64(c1.b)*(1-f) + float64(c2.b)*f) - - return fmt.Sprintf("rgb(%d, %d, %d)", r, g, b) -} - -func calculateSymbolSize(connections int) int { - if connections == 0 { - return 5 // Minimum size for nodes with no connections - } - // Use logarithmic scale to compress the range - logSize := math.Log1p(float64(connections)) // log(x+1) to handle 0 connections - // Map the log value to a range between 8 and 80 - minSize := 8.0 - maxSize := 80.0 - maxLogConnections := math.Log1p(1000) // Adjust this based on your max expected connections - scaledSize := minSize + math.Pow(logSize/maxLogConnections, 1.5)*(maxSize-minSize) - return int(math.Round(scaledSize)) - // return max(20, connections) -} diff --git a/pkg/graph/visualizer_test.go b/pkg/graph/visualizer_test.go deleted file mode 100644 index 099a8cd..0000000 --- a/pkg/graph/visualizer_test.go +++ /dev/null @@ -1,221 +0,0 @@ -package graph - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/RoaringBitmap/roaring" - "github.com/go-echarts/go-echarts/v2/opts" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestRunGraphVisualizer(t *testing.T) { - tests := []struct { - name string - setupMock func(*MockStorage) - ids *roaring.Bitmap - query string - addr string - expectError bool - }{ - { - name: "Successful visualization", - setupMock: func(ms *MockStorage) { - node := &Node{ID: 1, Name: "TestNode", Children: roaring.New(), Parents: roaring.New()} - ms.SaveNode(node) - node2 := &Node{ID: 2, Name: "TestNode2", Children: roaring.New(), Parents: roaring.New()} - ms.SaveNode(node2) - node3 := &Node{ID: 3, Name: "TestNode3", Children: roaring.New(), Parents: roaring.New()} - ms.SaveNode(node3) - node4 := &Node{ID: 4, Name: "TestNode4", Children: roaring.New(), Parents: roaring.New()} - ms.SaveNode(node4) - node.SetDependency(ms, node2) - node2.SetDependency(ms, node3) - node3.SetDependency(ms, node4) - }, - ids: roaring.BitmapOf(1), - query: "testdata query", - addr: "8081", - expectError: false, - }, - { - name: "Empty bitmap", - setupMock: func(ms *MockStorage) {}, - ids: roaring.New(), - query: "empty query", - addr: "8082", - expectError: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - http.DefaultServeMux = http.NewServeMux() - - mockStorage := NewMockStorage() - tt.setupMock(mockStorage) - - // Create a testdata server - testServer := httptest.NewServer(nil) - defer testServer.Close() - - server := &http.Server{ - Addr: testServer.Listener.Addr().String(), - } - - shutdown, err := RunGraphVisualizer(mockStorage, tt.ids, tt.query, server) - if tt.expectError { - assert.Error(t, err) - } else { - assert.NoError(t, err) - assert.NotNil(t, shutdown) - - // Test the shutdown function - done := make(chan struct{}) - go func() { - shutdown() - close(done) - }() - - select { - case <-done: - // Shutdown completed successfully - case <-time.After(5 * time.Second): - t.Fatal("Shutdown timed out") - } - } - }) - } -} - -func TestGraphQuery(t *testing.T) { - tests := []struct { - name string - setupMock func(*MockStorage) - ids *roaring.Bitmap - query string - expectError bool - nodeCount int - linkCount int - }{ - { - name: "Single node graph", - setupMock: func(ms *MockStorage) { - node := &Node{ID: 1, Name: "Node1", Children: roaring.New(), Parents: roaring.New()} - ms.SaveNode(node) - }, - ids: roaring.BitmapOf(1), - query: "single node", - expectError: false, - nodeCount: 1, - linkCount: 0, - }, - { - name: "Two connected nodes", - setupMock: func(ms *MockStorage) { - node1 := &Node{ID: 1, Name: "Node1", Children: roaring.BitmapOf(2), Parents: roaring.New()} - node2 := &Node{ID: 2, Name: "Node2", Children: roaring.New(), Parents: roaring.BitmapOf(1)} - ms.SaveNode(node1) - ms.SaveNode(node2) - }, - ids: roaring.BitmapOf(1, 2), - query: "two nodes", - expectError: false, - nodeCount: 2, - linkCount: 1, - }, - { - name: "Empty bitmap", - setupMock: func(ms *MockStorage) {}, - ids: roaring.New(), - query: "empty graph", - expectError: false, - nodeCount: 0, - linkCount: 0, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - http.DefaultServeMux = http.NewServeMux() - - mockStorage := NewMockStorage() - tt.setupMock(mockStorage) - - graph, err := graphQuery(mockStorage, tt.ids, tt.query) - if tt.expectError { - assert.Error(t, err) - assert.Nil(t, graph) - } else { - assert.NoError(t, err) - assert.NotNil(t, graph) - assert.Equal(t, tt.nodeCount, len(graph.MultiSeries[0].Data.([]opts.GraphNode))) - assert.Equal(t, tt.linkCount, len(graph.MultiSeries[0].Links.([]opts.GraphLink))) - } - }) - } -} - -func TestGraphVisualizerHTTPHandler(t *testing.T) { - tests := []struct { - name string - setupMock func(*MockStorage) - ids *roaring.Bitmap - query string - expectedCode int - }{ - { - name: "Successful render", - setupMock: func(ms *MockStorage) { - node := &Node{ID: 1, Name: "TestNode", Children: roaring.New(), Parents: roaring.New()} - ms.SaveNode(node) - node2 := &Node{ID: 2, Name: "TestNode2", Children: roaring.New(), Parents: roaring.New()} - ms.SaveNode(node2) - node3 := &Node{ID: 3, Name: "TestNode3", Children: roaring.New(), Parents: roaring.New()} - ms.SaveNode(node3) - node4 := &Node{ID: 4, Name: "TestNode4", Children: roaring.New(), Parents: roaring.New()} - ms.SaveNode(node4) - node.SetDependency(ms, node2) - node2.SetDependency(ms, node3) - node3.SetDependency(ms, node4) - }, - ids: roaring.BitmapOf(1), - query: "testdata query", - expectedCode: http.StatusOK, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - http.DefaultServeMux = http.NewServeMux() - - mockStorage := NewMockStorage() - tt.setupMock(mockStorage) - - req, err := http.NewRequest("GET", "/", nil) - require.NoError(t, err) - - rr := httptest.NewRecorder() - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - chart, err := graphQuery(mockStorage, tt.ids, tt.query) - if err != nil { - http.Error(w, "Error generating graph: "+err.Error(), http.StatusInternalServerError) - return - } - err = chart.Render(w) - if err != nil { - http.Error(w, "Error rendering graph: "+err.Error(), http.StatusInternalServerError) - return - } - }) - - handler.ServeHTTP(rr, req) - - assert.Equal(t, tt.expectedCode, rr.Code) - // You can add more assertions here to check the response body if needed - }) - } -} From d54778381c75e1a8059a0f093af7fc88efa176d4 Mon Sep 17 00:00:00 2001 From: Naveen <172697+naveensrinivasan@users.noreply.github.com> Date: Fri, 15 Nov 2024 10:01:37 -0600 Subject: [PATCH 7/7] Update Makefile Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1045966..f78d4ef 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,6 @@ check-wire: wire: check-wire cd cmd/server && wire || { echo "Wire generation failed in cmd/server"; exit 1; } -all: build test docker-build go-mod-tidy git-porcelain wire +all: wire build test docker-build go-mod-tidy git-porcelain .PHONY: test test-e2e build clean clean-redis docker-up docker-down docker-logs docker-build all wire \ No newline at end of file