Add log search API server (#345)

Makefile

@@ -12,7 +12,10 @@ KUSTOMIZE_CRDS=$(KUSTOMIZE_HOME)/crds/
 
 PLUGIN_HOME=kubectl-minio
 
-all: build
+LOGSEARCHAPI=logsearchapi
+LOGSEARCHAPI_TAG ?= "minio/logsearchapi:$(VERSION)"
+
+all: build logsearchapi
 
 getdeps:
 	@echo "Checking dependencies"
@@ -57,3 +60,15 @@ statik:
 plugin: regen-crd
 	@echo "Building 'kubectl-minio' binary"
 	@(cd $(PLUGIN_HOME); go build -o kubectl-minio main.go)
+
+.PHONY: logsearchapi
+logsearchapi:
+	@echo "Building 'logsearchapi' binary"
+	@(cd $(LOGSEARCHAPI); \
+		go vet ./... && \
+		go test -race ./... && \
+		GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean && \
+		GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ../.golangci.yml && \
+		go build && \
+		docker build -t $(LOGSEARCHAPI_TAG) . \
+   )

logsearchapi/Dockerfile

@@ -0,0 +1,7 @@
+FROM ubuntu:20.04
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+COPY logsearchapi /logsearchapi
+
+CMD ["/logsearchapi"]

logsearchapi/README.md

@@ -0,0 +1,33 @@
+# Log Search API Server for MinIO
+
+## Development setup
+
+1. Start Postgresql server in container:
+
+```shell
+docker run --rm -it -e "POSTGRES_PASSWORD=example" -p 5432:5432 postgres:13-alpine -c "log_statement=all"
+```
+
+2. Start logsearchapi server:
+
+```shell
+export LOGSEARCH_PG_CONN_STR="postgres://postgres:example@localhost/postgres"
+export LOGSEARCH_AUDIT_AUTH_TOKEN=xxx
+export LOGSEARCH_QUERY_AUTH_TOKEN=yyy
+export LOGSEARCH_MAX_RETENTION_MONTHS=3
+go build && ./logsearchapi
+```
+
+3. Minio setup:
+
+```shell
+mc admin config set myminio audit_webhook:1 'endpoint=http://localhost:8080/api/ingest?token=xxx'
+
+mc admin service restart myminio
+```
+
+4. Sample search/list queries:
+
+```shell
+curl -v "http://localhost:8080/api/query?token=yyy&q=raw&pageNo=0&pageSize=10&timeStart=2020-11-04T22:26:12.732402319Z"
+```

logsearchapi/go.mod

@@ -0,0 +1,10 @@
+module github.com/minio/operator/logsearchapi
+
+go 1.15
+
+require (
+	github.com/georgysavva/scany v0.2.7
+	github.com/gogo/protobuf v1.3.1
+	github.com/jackc/pgx/v4 v4.9.0
+	github.com/minio/minio v0.0.0-20201029200030-7331659d3ddc
+)

logsearchapi/main.go

@@ -0,0 +1,70 @@
+// +build go1.13
+
+/*
+ * Copyright (C) 2020, MinIO, Inc.
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+package main
+
+import (
+	"errors"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+
+	"github.com/minio/operator/logsearchapi/server"
+)
+
+const (
+	pgConnStrEnv          = "LOGSEARCH_PG_CONN_STR"
+	auditAuthTokenEnv     = "LOGSEARCH_AUDIT_AUTH_TOKEN"
+	queryAuthTokenEnv     = "LOGSEARCH_QUERY_AUTH_TOKEN"
+	maxRetentionMonthsEnv = "LOGSEARCH_MAX_RETENTION_MONTHS"
+)
+
+func loadEnv() (*server.LogSearch, error) {
+	pgConnStr := os.Getenv(pgConnStrEnv)
+	if pgConnStr == "" {
+		return nil, errors.New(pgConnStrEnv + " env variable is required.")
+	}
+	auditAuthToken := os.Getenv(auditAuthTokenEnv)
+	if auditAuthToken == "" {
+		return nil, errors.New(auditAuthTokenEnv + " env variable is required.")
+	}
+	queryAuthToken := os.Getenv(queryAuthTokenEnv)
+	if queryAuthToken == "" {
+		return nil, errors.New(queryAuthTokenEnv + " env variable is required.")
+	}
+	maxRetentionMonths, err := strconv.Atoi(os.Getenv(maxRetentionMonthsEnv))
+	if err != nil {
+		return nil, errors.New(maxRetentionMonthsEnv + " env variable is required and must be an integer.")
+	}
+
+	return server.NewLogSearch(pgConnStr, auditAuthToken, queryAuthToken, maxRetentionMonths)
+}
+
+func main() {
+	ls, err := loadEnv()
+	if err != nil {
+		log.Fatal(err)
+	}
+	s := &http.Server{
+		Addr:    ":8080",
+		Handler: ls,
+	}
+	log.Fatal(s.ListenAndServe())
+}

logsearchapi/server/auth.go

@@ -0,0 +1,36 @@
+// +build go1.13
+
+/*
+ * Copyright (C) 2020, MinIO, Inc.
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+package server
+
+import (
+	"crypto/subtle"
+	"net/http"
+)
+
+func authorize(h func(http.ResponseWriter, *http.Request), secretToken string) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		token := r.FormValue("token")
+		if subtle.ConstantTimeCompare([]byte(token), []byte(secretToken)) == 1 {
+			h(w, r)
+		} else {
+			w.WriteHeader(http.StatusForbidden)
+		}
+	}
+}