Promtail targets and service discovery pages (#532)

* add a custom server for loki with assets embedded

* adds a targets page for promtail to see discovered files and labels

* lint fixes

* fix css table size

* add service discovery logic

* add service discovery template

* add promtail pages documentation

* ignored gen file in linter

.gometalinter.json

@@ -0,0 +1,36 @@
+{
+  "Vendor": true,
+  "Deadline": "5m",
+  "Concurrency": 2,
+  "Linters": {
+    "gofmt": {"Command": "gofmt -l -s -w"},
+    "goimports": {"Command": "goimports -l -w"}
+  },
+  "Exclude": [
+    "\\.pb\\.go",
+    "pkg/promtail/server/ui/assets_vfsdata.go",
+    "method Seek.*should have signature",
+    "error return value not checked \\(level\\.",
+    "\"err\" shadows declaration"
+  ],
+
+  "Enable": [
+    "deadcode",
+    "errcheck",
+    "goconst",
+    "gofmt",
+    "goimports",
+    "golint",
+    "gosimple",
+    "gotypex",
+    "ineffassign",
+    "megacheck",
+    "misspell",
+    "structcheck",
+    "unconvert",
+    "unparam",
+    "varcheck",
+    "vet",
+    "vetshadow"
+  ]
+}

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all test clean images protos
+.PHONY: all test clean images protos assets check_assets
 .DEFAULT_GOAL := all
 
 CHARTS := production/helm/loki production/helm/promtail production/helm/loki-stack
@@ -90,6 +90,7 @@ vendor/github.com/cortexproject/cortex/pkg/ingester/client/cortex.pb.go: vendor/
 vendor/github.com/cortexproject/cortex/pkg/chunk/storage/caching_index_client.pb.go: vendor/github.com/cortexproject/cortex/pkg/chunk/storage/caching_index_client.proto
 pkg/parser/labels.go: pkg/parser/labels.y
 pkg/parser/matchers.go: pkg/parser/matchers.y
+pkg/promtail/server/ui/assets_vfsdata.go: assets
 all: $(UPTODATE_FILES)
 test: $(PROTO_GOS) $(YACC_GOS)
 debug: $(DEBUG_UPTODATE_FILES)
@@ -245,5 +246,16 @@ helm-publish: helm
 
 clean:
 	$(SUDO) docker rmi $(IMAGE_NAMES) $(DEBUG_IMAGE_NAMES) >/dev/null 2>&1 || true
-	rm -rf $(UPTODATE_FILES) $(EXES) $(DEBUG_UPTODATE_FILES) $(DEBUG_EXES) $(DEBUG_DLV_FILES) .cache
+	rm -rf $(UPTODATE_FILES) $(EXES) $(DEBUG_UPTODATE_FILES) $(DEBUG_EXES) $(DEBUG_DLV_FILES) .cache pkg/promtail/server/ui/assets_vfsdata.go
 	go clean ./...
+
+assets:
+	@echo ">> writing assets"
+	go generate -x -v ./pkg/promtail/server/ui
+
+check_assets: assets
+	@echo ">> checking that assets are up-to-date"
+	@if ! (cd pkg/promtail/server/ui && git diff --exit-code); then \
+		echo "Run 'make assets' and commit the changes to fix the error."; \
+		exit 1; \
+	fi

cmd/promtail/main.go

@@ -28,7 +28,7 @@ func main() {
 	flagext.RegisterFlags(&config)
 	flag.Parse()
 
-	util.InitLogger(&config.ServerConfig)
+	util.InitLogger(&config.ServerConfig.Config)
 
 	if configFile != "" {
 		if err := helpers.LoadConfig(configFile, &config); err != nil {

docs/troubleshooting.md

@@ -41,6 +41,18 @@ to
           target_label: __path__
 ```
 
+## Troubleshooting targets
+
+Promtail offers two pages that you can use to understand how service discovery works.
+The service discovery page (`/service-discovery`) shows all discovered targets with their labels before and after relabeling as well as the reason why the target has been dropped.
+The targets page (`/targets`) however displays only targets being actively scraped with their respective labels, files and positions.
+
+You can access those two pages by port-forwarding the promtail port (9080 or 3101 via helm) locally:
+
+```bash
+kubectl port-forward loki-promtail-jrfg7 9080
+```
+
 ## Debug output
 
 Both binaries support a log level parameter on the command-line, e.g.: `loki —log.level= debug ...`

pkg/promtail/config/config.go

@@ -3,11 +3,10 @@ package config
 import (
 	"flag"
 
-	"github.com/weaveworks/common/server"
-
 	"github.com/grafana/loki/pkg/promtail/client"
 	"github.com/grafana/loki/pkg/promtail/positions"
 	"github.com/grafana/loki/pkg/promtail/scrape"
+	"github.com/grafana/loki/pkg/promtail/server"
 	"github.com/grafana/loki/pkg/promtail/targets"
 )
 

pkg/promtail/promtail.go

@@ -1,14 +1,12 @@
 package promtail
 
 import (
-	"net/http"
-
 	"github.com/cortexproject/cortex/pkg/util"
-	"github.com/weaveworks/common/server"
 
 	"github.com/grafana/loki/pkg/promtail/client"
 	"github.com/grafana/loki/pkg/promtail/config"
 	"github.com/grafana/loki/pkg/promtail/positions"
+	"github.com/grafana/loki/pkg/promtail/server"
 	"github.com/grafana/loki/pkg/promtail/targets"
 )
 
@@ -37,19 +35,11 @@ func New(cfg config.Config) (*Promtail, error) {
 		return nil, err
 	}
 
-	server, err := server.New(cfg.ServerConfig)
+	server, err := server.New(cfg.ServerConfig, tms)
 	if err != nil {
 		return nil, err
 	}
 
-	server.HTTP.Path("/ready").Handler(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
-		if tms.Ready() {
-			rw.WriteHeader(http.StatusNoContent)
-		} else {
-			rw.WriteHeader(http.StatusInternalServerError)
-		}
-	}))
-
 	return &Promtail{
 		client:         client,
 		positions:      positions,