Log Search Tweaks and integration with Console (#391)

go.mod

@@ -5,8 +5,10 @@ go 1.13
 require (
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017
+	github.com/georgysavva/scany v0.2.7 // indirect
 	github.com/google/go-containerregistry v0.1.2
 	github.com/gorilla/mux v1.8.0
+	github.com/jackc/pgx/v4 v4.10.0 // indirect
 	github.com/minio/minio v0.0.0-20201203193910-919441d9c4d2
 	github.com/minio/minio-go/v7 v7.0.6
 	github.com/secure-io/sio-go v0.3.1 // indirect

logsearchapi/server/db.go

@@ -249,24 +249,26 @@ type logEventRawRow struct {
 	Log       string
 }
 
-type logEventRow struct {
-	EventTime time.Time
-	Log       map[string]interface{}
+// LogEventRow holds a raw log record
+type LogEventRow struct {
+	EventTime time.Time              `json:"event_time"`
+	Log       map[string]interface{} `json:"log"`
 }
 
-type reqInfoRow struct {
-	Time                  time.Time
-	APIName               string
-	Bucket                string
-	Object                string
-	TimeToResponseNs      uint64
-	RemoteHost            string
-	RequestID             string
-	UserAgent             string
-	ResponseStatus        string
-	ResponseStatusCode    int
-	RequestContentLength  *uint64
-	ResponseContentLength *uint64
+// ReqInfoRow holds a structured log record
+type ReqInfoRow struct {
+	Time                  time.Time `json:"time"`
+	APIName               string    `json:"api_name"`
+	Bucket                string    `json:"bucket"`
+	Object                string    `json:"object"`
+	TimeToResponseNs      uint64    `json:"time_to_response_ns"`
+	RemoteHost            string    `json:"remote_host"`
+	RequestID             string    `json:"request_id"`
+	UserAgent             string    `json:"user_agent"`
+	ResponseStatus        string    `json:"response_status"`
+	ResponseStatusCode    int       `json:"response_status_code"`
+	RequestContentLength  *uint64   `json:"request_content_length"`
+	ResponseContentLength *uint64   `json:"response_content_length"`
 }
 
 // Search executes a search query on the db.
@@ -299,26 +301,32 @@ func (c *DBClient) Search(ctx context.Context, s *SearchQuery, w io.Writer) erro
                                            OFFSET $1 LIMIT $2;`
 	)
 
-	timeRangeOp := "<="
 	timeOrder := "DESC"
 	if s.TimeAscending {
-		timeRangeOp = ">="
 		timeOrder = "ASC"
 	}
 
 	jw := json.NewEncoder(w)
 	switch s.Query {
 	case rawQ:
-		timeRangeClause := fmt.Sprintf("event_time %s '%s'", timeRangeOp, s.TimeStart.Format(time.RFC3339Nano))
-		q := logEventSelect.build(auditLogEventsTable.Name, timeRangeClause, timeOrder)
+		whereClauses := []string{}
+		// only filter by time if provided
+		if s.TimeStart != nil {
+			timeRangeOp := ">="
+			timeRangeClause := fmt.Sprintf("event_time %s '%s'", timeRangeOp, s.TimeStart.Format(time.RFC3339Nano))
+			whereClauses = append(whereClauses, timeRangeClause)
+		}
+		whereClause := strings.Join(whereClauses, " AND ")
+
+		q := logEventSelect.build(auditLogEventsTable.Name, whereClause, timeOrder)
 		rows, _ := c.Query(ctx, q, s.PageNumber*s.PageSize, s.PageSize)
 		var logEventsRaw []logEventRawRow
 		if err := pgxscan.ScanAll(&logEventsRaw, rows); err != nil {
 			return fmt.Errorf("Error accessing db: %v", err)
 		}
 		// parse the encoded json string stored in the db into a json
 		// object for output
-		logEvents := make([]logEventRow, len(logEventsRaw))
+		logEvents := make([]LogEventRow, len(logEventsRaw))
 		for i, e := range logEventsRaw {
 			logEvents[i].EventTime = e.EventTime
 			logEvents[i].Log = make(map[string]interface{})
@@ -333,21 +341,27 @@ func (c *DBClient) Search(ctx context.Context, s *SearchQuery, w io.Writer) erro
 		// For this query, $1 and $2 are used for offset and limit.
 		sqlArgs := []interface{}{s.PageNumber * s.PageSize, s.PageSize}
 
-		// $3 will be used for the time parameter
-		timeRangeClause := fmt.Sprintf("time %s $3", timeRangeOp)
-		whereClauses := []string{timeRangeClause}
-		sqlArgs = append(sqlArgs, s.TimeStart.Format(time.RFC3339Nano))
+		dollarStart := 3
+		whereClauses := []string{}
+		// only filter by time if provided
+		if s.TimeStart != nil {
+			// $3 will be used for the time parameter
+			timeRangeOp := ">="
+			timeRangeClause := fmt.Sprintf("time %s $%d", timeRangeOp, dollarStart)
+			sqlArgs = append(sqlArgs, s.TimeStart.Format(time.RFC3339Nano))
+			whereClauses = append(whereClauses, timeRangeClause)
+			dollarStart++
+		}
 
 		// Remaining dollar params are added for filter where clauses
-		filterClauses, filterArgs := generateFilterClauses(s.FParams, 4)
+		filterClauses, filterArgs := generateFilterClauses(s.FParams, dollarStart)
 		whereClauses = append(whereClauses, filterClauses...)
 		sqlArgs = append(sqlArgs, filterArgs...)
 
 		whereClause := strings.Join(whereClauses, " AND ")
 		q := reqInfoSelect.build(requestInfoTable.Name, whereClause, timeOrder)
-		// fmt.Println(q, sqlArgs)
 		rows, _ := c.Query(ctx, q, sqlArgs...)
-		var reqInfos []reqInfoRow
+		var reqInfos []ReqInfoRow
 		err := pgxscan.ScanAll(&reqInfos, rows)
 		if err != nil {
 			return fmt.Errorf("Error accessing db: %v", err)

logsearchapi/server/query.go

@@ -51,7 +51,7 @@ func stringToFParam(s string) (f fParam, err error) {
 // SearchQuery represents a search query.
 type SearchQuery struct {
 	Query         qType
-	TimeStart     time.Time
+	TimeStart     *time.Time
 	TimeAscending bool
 	PageNumber    int
 	PageSize      int
@@ -92,14 +92,13 @@ func searchQueryFromRequest(r *http.Request) (*SearchQuery, error) {
 		return nil, fmt.Errorf("Invalid query name: %s", string(q))
 	}
 
-	var timeStart time.Time
+	var timeStart *time.Time
 	if timeParam := values.Get("timeStart"); timeParam != "" {
-		timeStart, err = parseSQTimeString(timeParam)
+		ts, err := parseSQTimeString(timeParam)
 		if err != nil {
 			return nil, fmt.Errorf("Invalid start date (must be RFC3339 format): %s", timeParam)
 		}
-	} else {
-		timeStart = time.Now()
+		timeStart = &ts
 	}
 
 	var pageSize int = 10

pkg/controller/cluster/console-csr.go → pkg/controller/cluster/console.go

@@ -23,6 +23,12 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
+	"errors"
+
+	"github.com/minio/operator/pkg/resources/deployments"
+	"k8s.io/apimachinery/pkg/api/equality"
+
+	appsv1 "k8s.io/api/apps/v1"
 
 	miniov1 "github.com/minio/operator/pkg/apis/minio.min.io/v1"
 
@@ -94,3 +100,26 @@ func (c *Controller) createConsoleTLSCSR(ctx context.Context, tenant *miniov1.Te
 
 	return nil
 }
+
+// consoleDeploymentMatchesSpec checks if the deployment for console matches what is expected and described from the Tenant
+func consoleDeploymentMatchesSpec(tenant *miniov1.Tenant, consoleDeployment *appsv1.Deployment) (bool, error) {
+	if consoleDeployment == nil {
+		return false, errors.New("cannot process an empty console deployment")
+	}
+	if tenant == nil {
+		return false, errors.New("cannot process an empty tenant")
+	}
+	// compare image directly
+	if !tenant.Spec.Console.EqualImage(consoleDeployment.Spec.Template.Spec.Containers[0].Image) {
+		klog.V(2).Infof("Tenant %s console version %s doesn't match: %s", tenant.Name,
+			tenant.Spec.Console.Image, consoleDeployment.Spec.Template.Spec.Containers[0].Image)
+		return false, nil
+	}
+	// compare any other change from what is specified on the tenant
+	expectedDeployment := deployments.NewConsole(tenant)
+	if !equality.Semantic.DeepDerivative(expectedDeployment.Spec, consoleDeployment.Spec) {
+		// some field set by the operator has changed
+		return false, nil
+	}
+	return true, nil
+}