Fixes race on shutdown and adds documentation

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>

cmd/promtail/main.go

@@ -64,11 +64,10 @@ func main() {
 	}
 
 	level.Info(util.Logger).Log("msg", "Starting Promtail", "version", version.Info())
+	defer p.Shutdown()
 
 	if err := p.Run(); err != nil {
 		level.Error(util.Logger).Log("msg", "error starting promtail", "error", err)
 		os.Exit(1)
 	}
-
-	p.Shutdown()
 }

docs/clients/promtail/troubleshooting.md

@@ -3,6 +3,53 @@
 This document describes known failure modes of `promtail` on edge cases and the
 adopted trade-offs.
 
+## Pipe data to Promtail
+
+Promtail supports piping data for sending logs to Loki. This is a very useful way to troubleshooting your configuration.
+Once you have promtail installed you can for instance use the following command to send logs to a local Loki instance:
+
+```bash
+cat my.log | promtail --client.url http://127.0.0.1:3100/loki/api/v1/push
+```
+
+You can also add additional labels from command line using:
+
+```bash
+cat my.log | promtail --client.url http://127.0.0.1:3100/loki/api/v1/push --client.external-labels=k1=v1,k2=v2
+```
+
+This will add labels `k1` and `k2` with respective values `v1` and `v2`.
+
+In pipe mode Promtail also support file configuration using `--config.file`, however do note that positions config is not used and
+only **the first scrape config is used**.
+
+[`static_configs:`](./configuration) can be used to provide static labels, although the targets property is ignored.
+
+If you don't provide any [`scrape_config:`](./configuration#scrape_config) a default one is used which will automatically adds the following default labels: `{job="stdin",hostname="Cyrils-iMac"}`.
+
+For example you could use this config below to parse and add the label `level` on all your piped logs:
+
+```yaml
+clients:
+  - url: http://localhost:3100/loki/api/v1/push
+
+scrape_configs:
+- job_name: system
+  pipeline_stages:
+  - regex:
+      expression: '(level|lvl|severity)=(?P<level>\\w+)'
+  - labels:
+      level:
+  static_configs:
+  - labels:
+      job: my-stdin-logs
+```
+
+```
+cat my.log | promtail --config.file promtail.yaml
+```
+
+
 ## A tailed file is truncated while `promtail` is not running
 
 Given the following order of events:

pkg/promtail/promtail.go

@@ -1,29 +1,28 @@
 package promtail
 
 import (
+	"sync"
+
 	"github.com/cortexproject/cortex/pkg/util"
 
 	"github.com/grafana/loki/pkg/promtail/client"
 	"github.com/grafana/loki/pkg/promtail/config"
-	"github.com/grafana/loki/pkg/promtail/positions"
 	"github.com/grafana/loki/pkg/promtail/server"
 	"github.com/grafana/loki/pkg/promtail/targets"
 )
 
 // Promtail is the root struct for Promtail...
 type Promtail struct {
 	client         client.Client
-	positions      *positions.Positions
 	targetManagers *targets.TargetManagers
 	server         *server.Server
+
+	stopped bool
+	mtx     sync.Mutex
 }
 
 // New makes a new Promtail.
 func New(cfg config.Config) (*Promtail, error) {
-	positions, err := positions.New(util.Logger, cfg.PositionsConfig)
-	if err != nil {
-		return nil, err
-	}
 
 	if cfg.ClientConfig.URL.URL != nil {
 		// if a single client config is used we add it to the multiple client config for backward compatibility
@@ -36,11 +35,10 @@ func New(cfg config.Config) (*Promtail, error) {
 	}
 
 	promtail := &Promtail{
-		client:    client,
-		positions: positions,
+		client: client,
 	}
 
-	tms, err := targets.NewTargetManagers(promtail, util.Logger, positions, client, cfg.ScrapeConfig, &cfg.TargetConfig)
+	tms, err := targets.NewTargetManagers(promtail, util.Logger, cfg.PositionsConfig, client, cfg.ScrapeConfig, &cfg.TargetConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -55,17 +53,25 @@ func New(cfg config.Config) (*Promtail, error) {
 
 // Run the promtail; will block until a signal is received.
 func (p *Promtail) Run() error {
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+	// if we stopped promtail before the server even started we can return without starting.
+	if p.stopped {
+		return nil
+	}
 	return p.server.Run()
 }
 
 // Shutdown the promtail.
 func (p *Promtail) Shutdown() {
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+	p.stopped = true
 	if p.server != nil {
 		p.server.Shutdown()
 	}
 	if p.targetManagers != nil {
 		p.targetManagers.Stop()
 	}
-	p.positions.Stop()
 	p.client.Stop()
 }

pkg/promtail/targets/manager.go

@@ -1,6 +1,7 @@
 package targets
 
 import (
+	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/go-kit/kit/log"
 	"github.com/pkg/errors"
 
@@ -19,13 +20,14 @@ type targetManager interface {
 // TargetManagers manages a list of target managers.
 type TargetManagers struct {
 	targetManagers []targetManager
+	positions      *positions.Positions
 }
 
 // NewTargetManagers makes a new TargetManagers
 func NewTargetManagers(
 	app Shutdownable,
 	logger log.Logger,
-	positions *positions.Positions,
+	positionsConfig positions.Config,
 	client api.EntryHandler,
 	scrapeConfigs []scrape.Config,
 	targetConfig *Config,
@@ -44,6 +46,11 @@ func NewTargetManagers(
 		return &TargetManagers{targetManagers: targetManagers}, nil
 	}
 
+	positions, err := positions.New(util.Logger, positionsConfig)
+	if err != nil {
+		return nil, err
+	}
+
 	for _, cfg := range scrapeConfigs {
 		if cfg.HasServiceDiscoveryConfig() {
 			fileScrapeConfigs = append(fileScrapeConfigs, cfg)
@@ -94,7 +101,10 @@ func NewTargetManagers(
 		targetManagers = append(targetManagers, syslogTargetManager)
 	}
 
-	return &TargetManagers{targetManagers: targetManagers}, nil
+	return &TargetManagers{
+		targetManagers: targetManagers,
+		positions:      positions,
+	}, nil
 
 }
 
@@ -135,4 +145,7 @@ func (tm *TargetManagers) Stop() {
 	for _, t := range tm.targetManagers {
 		t.Stop()
 	}
+	if tm.positions != nil {
+		tm.positions.Stop()
+	}
 }