Lint, vet, format

.github/workflows/go.yml

@@ -24,6 +24,12 @@ jobs:
     - name: Build
       run: go build -v ./...
 
+    - name: golangci-lint
+      uses: golangci/golangci-lint-action@v3
+      with:
+        version: v1.55.2
+        args: --verbose --timeout 10m --fix=false --config=.golangci.yml ./pkg/...
+
     - name: Generate coverage report for coveralls.io
       run: |
         go test -coverprofile=/var/tmp/capillaries.p.tmp -cover $(find ./ -name '*_test.go' -printf "%h\n" | sort -u)
@@ -75,7 +81,7 @@ jobs:
 
     - name: pkg/sc test coverage threshold check
       env:
-          TESTCOVERAGE_THRESHOLD: 90.8
+          TESTCOVERAGE_THRESHOLD: 91.5
       run: |
             go test -v ./pkg/sc/... -coverprofile coverage.out -covermode count
             totalCoverage=`go tool cover -func=coverage.out | grep total | grep -Eo '[0-9]+\.[0-9]+'`
@@ -88,7 +94,7 @@ jobs:
 
     - name: pkg/custom/py_calc test coverage threshold check
       env:
-          TESTCOVERAGE_THRESHOLD: 81.3
+          TESTCOVERAGE_THRESHOLD: 83.3
       run: |
             go test -v ./pkg/custom/py_calc/... -coverprofile coverage.out -covermode count
             totalCoverage=`go tool cover -func=coverage.out | grep total | grep -Eo '[0-9]+\.[0-9]+'`

.golangci.yml

@@ -0,0 +1,119 @@
+# https://golangci-lint.run/usage/configuration/#config-file
+linters:
+  disable-all: true
+  enable:
+#    - goerr113
+    - errcheck
+#    - goimports
+    # - paralleltest # missing the call to method parallel, but testify does not seem to work well with parallel test: https://github.com/stretchr/testify/issues/187
+    - revive # revive supersedes golint, which is now archived
+    - staticcheck
+    - vet
+#    - forbidigo
+run:
+#  skip-dirs:
+#    - ^api
+#    - ^proto
+#    - ^.git
+linters-settings:
+#  govet:
+#    fieldalignment: 0
+#  forbidigo:
+#    forbid:
+#      - p: ^time\.After$
+#        msg: "time.After may leak resources. Use time.NewTimer instead."
+   revive:
+     severity: error
+     confidence: 0.8
+     enable-all-rules: true
+     rules:
+       # Disabled rules
+       - name: confusing-results
+         disabled: true
+       - name: add-constant
+         disabled: true
+       - name: argument-limit
+         disabled: true
+#       - name: bare-return
+#         disabled: true
+#       - name: banned-characters
+#         disabled: true
+#       - name: bool-literal-in-expr
+#         disabled: true
+#       - name: confusing-naming
+#         disabled: true
+       - name: empty-lines
+         disabled: true
+#       - name: error-naming
+#         disabled: true
+#       - name: errorf
+#         disabled: true
+       - name: exported
+         disabled: true
+#       - name: file-header
+#         disabled: true
+       - name: function-length
+         disabled: true
+#       - name: imports-blacklist
+#         disabled: true
+#       - name: increment-decrement
+#         disabled: true
+       - name: line-length-limit
+         disabled: true
+       - name: max-public-structs
+         disabled: true
+#       - name: nested-structs
+#         disabled: true
+#       - name: package-comments
+#         disabled: true
+#       - name: string-format
+#         disabled: true
+#       - name: unexported-naming
+#         disabled: true
+#       - name: unexported-return
+#         disabled: true
+#       - name: unused-parameter
+#         disabled: true
+       - name: unused-receiver
+         disabled: true
+#       - name: use-any
+#         disabled: true
+       - name: var-naming
+         disabled: true
+#       - name: empty-block
+#         disabled: true
+       - name: flag-parameter
+         disabled: true
+
+       # Rule tuning
+       - name: cognitive-complexity
+         arguments:
+           - 400 # TODO: do something
+       - name: cyclomatic
+         arguments:
+           - 100
+       - name: function-result-limit
+         arguments:
+           - 4
+       - name: unhandled-error
+         arguments:
+           - "fmt.*"
+           - "bytes.Buffer.*"
+           - "strings.Builder.*"
+           - "os.File.Close"
+           - "io.Closer.Close"
+           - "zap.Logger.Sync*"
+# issues:
+#   # Exclude cyclomatic and cognitive complexity rules for functional tests in the `tests` root directory.
+#   exclude-rules:
+#     - path: ^tests\/.+\.go
+#       text: "(cyclomatic|cognitive)"
+#       linters:
+#         - revive
+#     - path: _test\.go|^common/persistence\/tests\/.+\.go # Ignore things like err = errors.New("test error") in tests
+#       linters:
+#         - goerr113
+#     - path: ^tools\/.+\.go
+#       linters:
+#         - goerr113
+#         - revive

.vscode/settings.json

@@ -1,3 +1,4 @@
 {
-    "git.ignoreLimitWarning": true
+    "git.ignoreLimitWarning": true,
+    "editor.insertSpaces": false
 }

README.md

@@ -1,4 +1,4 @@
-# <img src="doc/logo.svg" alt="logo" width="60"/> Capillaries <div style="float:right;"><a href='https://coveralls.io/github/capillariesio/capillaries?branch=main'><img src='https://coveralls.io/repos/github/capillariesio/capillaries/badge.svg?branch=main' alt='Coverage Status' /></a></div>
+# <img src="doc/logo.svg" alt="logo" width="60"/> Capillaries <div style="float:right;"> [![coveralls](https://coveralls.io/repos/github/capillariesio/capillaries/badge.svg?branch=main)](https://coveralls.io/github/capillariesio/capillaries?branch=main) [![goreport](https://goreportcard.com/badge/github.com/capillariesio/capillaries)](https://goreportcard.com/report/github.com/capillariesio/capillaries)</div>
 
 
 Capillaries is a data processing framework that:

pkg/api/db.go

@@ -1,117 +1,114 @@
-package api
-
-import (
-	"fmt"
-	"reflect"
-	"regexp"
-	"strings"
-
-	"github.com/capillariesio/capillaries/pkg/cql"
-	"github.com/capillariesio/capillaries/pkg/db"
-	"github.com/capillariesio/capillaries/pkg/l"
-	"github.com/capillariesio/capillaries/pkg/proc"
-	"github.com/capillariesio/capillaries/pkg/sc"
-	"github.com/capillariesio/capillaries/pkg/wfdb"
-	"github.com/capillariesio/capillaries/pkg/wfmodel"
-	"github.com/gocql/gocql"
-)
-
-const ProhibitedKeyspaceNameRegex = "^system"
-const AllowedKeyspaceNameRegex = "[a-zA-Z0-9_]+"
-
-func IsSystemKeyspaceName(keyspace string) bool {
-	re := regexp.MustCompile(ProhibitedKeyspaceNameRegex)
-	invalidNamePieceFound := re.FindString(keyspace)
-	if len(invalidNamePieceFound) > 0 {
-		return true
-	}
-	return false
-}
-
-func checkKeyspaceName(keyspace string) error {
-	re := regexp.MustCompile(ProhibitedKeyspaceNameRegex)
-	invalidNamePieceFound := re.FindString(keyspace)
-	if len(invalidNamePieceFound) > 0 {
-		return fmt.Errorf("invalid keyspace name [%s]: prohibited regex is [%s]", keyspace, ProhibitedKeyspaceNameRegex)
-	}
-	re = regexp.MustCompile(AllowedKeyspaceNameRegex)
-	if !re.MatchString(keyspace) {
-		return fmt.Errorf("invalid keyspace name [%s]: allowed regex is [%s]", keyspace, AllowedKeyspaceNameRegex)
-	}
-	return nil
-}
-
-// A helper used by Toolbelt get_table_cql cmd, no logging needed
-func GetTablesCql(script *sc.ScriptDef, keyspace string, runId int16, startNodeNames []string) string {
-	sb := strings.Builder{}
-	sb.WriteString("-- Workflow\n")
-	sb.WriteString(fmt.Sprintf("%s\n", wfmodel.GetCreateTableCql(reflect.TypeOf(wfmodel.BatchHistoryEvent{}), keyspace, wfmodel.TableNameBatchHistory)))
-	sb.WriteString(fmt.Sprintf("%s\n", wfmodel.GetCreateTableCql(reflect.TypeOf(wfmodel.NodeHistoryEvent{}), keyspace, wfmodel.TableNameNodeHistory)))
-	sb.WriteString(fmt.Sprintf("%s\n", wfmodel.GetCreateTableCql(reflect.TypeOf(wfmodel.RunHistoryEvent{}), keyspace, wfmodel.TableNameRunHistory)))
-	sb.WriteString(fmt.Sprintf("%s\n", wfmodel.GetCreateTableCql(reflect.TypeOf(wfmodel.RunProperties{}), keyspace, wfmodel.TableNameRunAffectedNodes)))
-	sb.WriteString(fmt.Sprintf("%s\n", wfmodel.GetCreateTableCql(reflect.TypeOf(wfmodel.RunCounter{}), keyspace, wfmodel.TableNameRunCounter)))
-	qb := cql.QueryBuilder{}
-	sb.WriteString(fmt.Sprintf("%s\n", qb.Keyspace(keyspace).Write("ks", keyspace).Write("last_run", 0).InsertUnpreparedQuery(wfmodel.TableNameRunCounter, cql.IgnoreIfExists)))
-
-	for _, nodeName := range script.GetAffectedNodes(startNodeNames) {
-		node, ok := script.ScriptNodes[nodeName]
-		if !ok || !node.HasTableCreator() {
-			continue
-		}
-		sb.WriteString(fmt.Sprintf("-- %s\n", nodeName))
-		sb.WriteString(fmt.Sprintf("%s\n", proc.CreateDataTableCql(keyspace, runId, &node.TableCreator)))
-		for idxName, idxDef := range node.TableCreator.Indexes {
-			sb.WriteString(fmt.Sprintf("%s\n", proc.CreateIdxTableCql(keyspace, runId, idxName, idxDef)))
-		}
-	}
-	return sb.String()
-}
-
-// Used by Toolbelt and Webapi
-func DropKeyspace(logger *l.Logger, cqlSession *gocql.Session, keyspace string) error {
-	logger.PushF("api.DropKeyspace")
-	defer logger.PopF()
-
-	if err := checkKeyspaceName(keyspace); err != nil {
-		return err
-	}
-
-	qb := cql.QueryBuilder{}
-	q := qb.
-		Keyspace(keyspace).
-		DropKeyspace()
-	if err := cqlSession.Query(q).Exec(); err != nil {
-		return db.WrapDbErrorWithQuery("cannot drop keyspace", q, err)
-	}
-	return nil
-}
-
-// wfdb wrapper for webapi use
-func HarvestRunLifespans(logger *l.Logger, cqlSession *gocql.Session, keyspace string, runIds []int16) (wfmodel.RunLifespanMap, error) {
-	logger.PushF("api.HarvestRunLifespans")
-	defer logger.PopF()
-
-	return wfdb.HarvestRunLifespans(logger, cqlSession, keyspace, runIds)
-}
-
-// wfdb wrapper for webapi use
-func GetRunProperties(logger *l.Logger, cqlSession *gocql.Session, keyspace string, runId int16) ([]*wfmodel.RunProperties, error) {
-	logger.PushF("api.GetRunProperties")
-	defer logger.PopF()
-	return wfdb.GetRunProperties(logger, cqlSession, keyspace, runId)
-}
-
-// wfdb wrapper for webapi use
-func GetNodeHistoryForRun(logger *l.Logger, cqlSession *gocql.Session, keyspace string, runId int16) ([]*wfmodel.NodeHistoryEvent, error) {
-	logger.PushF("api.GetNodeHistoryForRun")
-	defer logger.PopF()
-
-	return wfdb.GetNodeHistoryForRun(logger, cqlSession, keyspace, runId)
-}
-
-// wfdb wrapper for webapi use
-func GetRunNodeBatchHistory(logger *l.Logger, cqlSession *gocql.Session, keyspace string, runId int16, nodeName string) ([]*wfmodel.BatchHistoryEvent, error) {
-	logger.PushF("api.GetRunNodeBatchHistory")
-	defer logger.PopF()
-	return wfdb.GetRunNodeBatchHistory(logger, cqlSession, keyspace, runId, nodeName)
-}
+package api
+
+import (
+	"fmt"
+	"reflect"
+	"regexp"
+	"strings"
+
+	"github.com/capillariesio/capillaries/pkg/cql"
+	"github.com/capillariesio/capillaries/pkg/db"
+	"github.com/capillariesio/capillaries/pkg/l"
+	"github.com/capillariesio/capillaries/pkg/proc"
+	"github.com/capillariesio/capillaries/pkg/sc"
+	"github.com/capillariesio/capillaries/pkg/wfdb"
+	"github.com/capillariesio/capillaries/pkg/wfmodel"
+	"github.com/gocql/gocql"
+)
+
+const ProhibitedKeyspaceNameRegex = "^system"
+const AllowedKeyspaceNameRegex = "[a-zA-Z0-9_]+"
+
+func IsSystemKeyspaceName(keyspace string) bool {
+	re := regexp.MustCompile(ProhibitedKeyspaceNameRegex)
+	invalidNamePieceFound := re.FindString(keyspace)
+	return len(invalidNamePieceFound) > 0
+}
+
+func checkKeyspaceName(keyspace string) error {
+	re := regexp.MustCompile(ProhibitedKeyspaceNameRegex)
+	invalidNamePieceFound := re.FindString(keyspace)
+	if len(invalidNamePieceFound) > 0 {
+		return fmt.Errorf("invalid keyspace name [%s]: prohibited regex is [%s]", keyspace, ProhibitedKeyspaceNameRegex)
+	}
+	re = regexp.MustCompile(AllowedKeyspaceNameRegex)
+	if !re.MatchString(keyspace) {
+		return fmt.Errorf("invalid keyspace name [%s]: allowed regex is [%s]", keyspace, AllowedKeyspaceNameRegex)
+	}
+	return nil
+}
+
+// A helper used by Toolbelt get_table_cql cmd, no logging needed
+func GetTablesCql(script *sc.ScriptDef, keyspace string, runId int16, startNodeNames []string) string {
+	sb := strings.Builder{}
+	sb.WriteString("-- Workflow\n")
+	sb.WriteString(fmt.Sprintf("%s\n", wfmodel.GetCreateTableCql(reflect.TypeOf(wfmodel.BatchHistoryEvent{}), keyspace, wfmodel.TableNameBatchHistory)))
+	sb.WriteString(fmt.Sprintf("%s\n", wfmodel.GetCreateTableCql(reflect.TypeOf(wfmodel.NodeHistoryEvent{}), keyspace, wfmodel.TableNameNodeHistory)))
+	sb.WriteString(fmt.Sprintf("%s\n", wfmodel.GetCreateTableCql(reflect.TypeOf(wfmodel.RunHistoryEvent{}), keyspace, wfmodel.TableNameRunHistory)))
+	sb.WriteString(fmt.Sprintf("%s\n", wfmodel.GetCreateTableCql(reflect.TypeOf(wfmodel.RunProperties{}), keyspace, wfmodel.TableNameRunAffectedNodes)))
+	sb.WriteString(fmt.Sprintf("%s\n", wfmodel.GetCreateTableCql(reflect.TypeOf(wfmodel.RunCounter{}), keyspace, wfmodel.TableNameRunCounter)))
+	qb := cql.QueryBuilder{}
+	sb.WriteString(fmt.Sprintf("%s\n", qb.Keyspace(keyspace).Write("ks", keyspace).Write("last_run", 0).InsertUnpreparedQuery(wfmodel.TableNameRunCounter, cql.IgnoreIfExists)))
+
+	for _, nodeName := range script.GetAffectedNodes(startNodeNames) {
+		node, ok := script.ScriptNodes[nodeName]
+		if !ok || !node.HasTableCreator() {
+			continue
+		}
+		sb.WriteString(fmt.Sprintf("-- %s\n", nodeName))
+		sb.WriteString(fmt.Sprintf("%s\n", proc.CreateDataTableCql(keyspace, runId, &node.TableCreator)))
+		for idxName, idxDef := range node.TableCreator.Indexes {
+			sb.WriteString(fmt.Sprintf("%s\n", proc.CreateIdxTableCql(keyspace, runId, idxName, idxDef)))
+		}
+	}
+	return sb.String()
+}
+
+// Used by Toolbelt and Webapi
+func DropKeyspace(logger *l.CapiLogger, cqlSession *gocql.Session, keyspace string) error {
+	logger.PushF("api.DropKeyspace")
+	defer logger.PopF()
+
+	if err := checkKeyspaceName(keyspace); err != nil {
+		return err
+	}
+
+	qb := cql.QueryBuilder{}
+	q := qb.
+		Keyspace(keyspace).
+		DropKeyspace()
+	if err := cqlSession.Query(q).Exec(); err != nil {
+		return db.WrapDbErrorWithQuery("cannot drop keyspace", q, err)
+	}
+	return nil
+}
+
+// wfdb wrapper for webapi use
+func HarvestRunLifespans(logger *l.CapiLogger, cqlSession *gocql.Session, keyspace string, runIds []int16) (wfmodel.RunLifespanMap, error) {
+	logger.PushF("api.HarvestRunLifespans")
+	defer logger.PopF()
+
+	return wfdb.HarvestRunLifespans(logger, cqlSession, keyspace, runIds)
+}
+
+// wfdb wrapper for webapi use
+func GetRunProperties(logger *l.CapiLogger, cqlSession *gocql.Session, keyspace string, runId int16) ([]*wfmodel.RunProperties, error) {
+	logger.PushF("api.GetRunProperties")
+	defer logger.PopF()
+	return wfdb.GetRunProperties(logger, cqlSession, keyspace, runId)
+}
+
+// wfdb wrapper for webapi use
+func GetNodeHistoryForRun(logger *l.CapiLogger, cqlSession *gocql.Session, keyspace string, runId int16) ([]*wfmodel.NodeHistoryEvent, error) {
+	logger.PushF("api.GetNodeHistoryForRun")
+	defer logger.PopF()
+
+	return wfdb.GetNodeHistoryForRun(logger, cqlSession, keyspace, runId)
+}
+
+// wfdb wrapper for webapi use
+func GetRunNodeBatchHistory(logger *l.CapiLogger, cqlSession *gocql.Session, keyspace string, runId int16, nodeName string) ([]*wfmodel.BatchHistoryEvent, error) {
+	logger.PushF("api.GetRunNodeBatchHistory")
+	defer logger.PopF()
+	return wfdb.GetRunNodeBatchHistory(logger, cqlSession, keyspace, runId, nodeName)
+}