Update: implement an adhoc scraper

.github/workflows/goreleaser.yml

@@ -13,9 +13,10 @@ jobs:
  - uses: actions/checkout@v3
  with:
  fetch-depth: 0
- - uses: actions/setup-go@v3
+ - uses: actions/setup-go@v4
  with:
  go-version-file: go.mod
+ cache: false
  - uses: docker/setup-qemu-action@v2
  - uses: docker/setup-buildx-action@v2
  - uses: docker/login-action@v1

.github/workflows/integration.yml

@@ -13,9 +13,10 @@ jobs:
  with:
  fetch-depth: 0
  - name: Set up Go
- uses: actions/setup-go@v3
+ uses: actions/setup-go@v4
  with:
  go-version-file: go.mod
+ cache: false
  - name: Run GoReleaser
  uses: goreleaser/goreleaser-action@v2
  with:

.github/workflows/pr.yml

@@ -7,23 +7,25 @@ jobs:
  test:
  strategy:
  matrix:
- go-version: [1.19.x, 1.20.x]
+ go-version: [1.20.x]
  runs-on: ubuntu-latest
  steps:
  - uses: actions/checkout@v3
- - uses: actions/setup-go@v3
+ - uses: actions/setup-go@v4
  with:
  go-version: ${{ matrix.go-version }}
+ cache: false
  - run: go test ./...
 
  release_test:
  needs: test
  runs-on: ubuntu-latest
  steps:
  - uses: actions/checkout@v3
- - uses: actions/setup-go@v3
+ - uses: actions/setup-go@v4
  with:
  go-version-file: go.mod
+ cache: false
  - uses: docker/setup-qemu-action@v2
  - uses: docker/setup-buildx-action@v2
  - run: docker buildx ls

Makefile

@@ -1,19 +1,15 @@
-GO_VERSION:=1.16
-
 .PHONY: build
 build:
- goreleaser build --snapshot --single-target --rm-dist
+ goreleaser build --snapshot --single-target --clean
+
+.PHONY: run-dev
+run-dev:
+ go run dnsbl_exporter.go --log.debug
 
 .PHONY: snapshot
 snapshot:
- goreleaser build --snapshot --rm-dist
+ goreleaser build --snapshot --clean
 
 .PHONY: test
 test:
- docker run \
- -it \
- --rm \
- -v $(CURDIR):/src/github.com/Luzilla/dnsbl_exporter \
- -w /src/github.com/Luzilla/dnsbl_exporter \
- golang:$(GO_VERSION) \
- sh -c "go mod download && go test ./..."
+ act "pull_request" -j test

app/app.go

@@ -1,153 +1,195 @@
 package app
 
 import (
+ "errors"
+ "fmt"
+ "io"
  "net/http"
  "os"
+ "strings"
 
- "github.com/Luzilla/dnsbl_exporter/collector"
  "github.com/Luzilla/dnsbl_exporter/config"
+ "github.com/Luzilla/dnsbl_exporter/internal/index"
+ "github.com/Luzilla/dnsbl_exporter/internal/metrics"
+ "github.com/Luzilla/dnsbl_exporter/internal/prober"
+ "github.com/Luzilla/dnsbl_exporter/internal/setup"
  "github.com/prometheus/client_golang/prometheus"
- "github.com/prometheus/client_golang/prometheus/promhttp"
- "github.com/urfave/cli"
-
- log "github.com/sirupsen/logrus"
+ "github.com/urfave/cli/v2"
+ "golang.org/x/exp/slog"
 )
 
 type DNSBLApp struct {
  App *cli.App
 }
 
+var (
+ appName, appVersion, appPath string
+ resolver string
+)
+
 // NewApp ...
 func NewApp(name string, version string) DNSBLApp {
-
- cli.VersionFlag = cli.BoolFlag{
- Name: "version, V",
- Usage: "Print the version information.",
- }
-
- app := cli.NewApp()
- app.Name = name
- app.Version = version
- app.Flags = []cli.Flag{
- cli.StringFlag{
- Name: "config.dns-resolver",
- Value: "127.0.0.1:53",
- Usage: "IP address[:port] of the resolver to use.",
- EnvVar: "DNSBL_EXP_RESOLVER",
+ appName = name
+ appVersion = version
+
+ a := cli.NewApp()
+ a.Name = appName
+ a.Version = appVersion
+ a.Flags = []cli.Flag{
+ &cli.StringFlag{
+ Name: "config.dns-resolver",
+ Value: "127.0.0.1:53",
+ Usage: "IP address[:port] of the resolver to use.",
+ EnvVars: []string{"DNSBL_EXP_RESOLVER"},
+ Destination: &resolver,
  },
- cli.StringFlag{
- Name: "config.rbls",
- Value: "./rbls.ini",
- Usage: "Configuration file which contains RBLs",
- EnvVar: "DNSBL_EXP_RBLS",
+ &cli.StringFlag{
+ Name:  "config.rbls",
+ Value:  "./rbls.ini",
+ Usage:  "Configuration file which contains RBLs",
+ EnvVars: []string{"DNSBL_EXP_RBLS"},
  },
- cli.StringFlag{
- Name: "config.targets",
- Value: "./targets.ini",
- Usage: "Configuration file which contains the targets to check.",
- EnvVar: "DNSBL_EXP_TARGETS",
+ &cli.StringFlag{
+ Name:  "config.targets",
+ Value:  "./targets.ini",
+ Usage:  "Configuration file which contains the targets to check.",
+ EnvVars: []string{"DNSBL_EXP_TARGETS"},
  },
- cli.StringFlag{
- Name: "web.listen-address",
- Value: ":9211",
- Usage: "Address to listen on for web interface and telemetry.",
- EnvVar: "DNSBL_EXP_LISTEN",
+ &cli.StringFlag{
+ Name:  "web.listen-address",
+ Value:  ":9211",
+ Usage:  "Address to listen on for web interface and telemetry.",
+ EnvVars: []string{"DNSBL_EXP_LISTEN"},
  },
- cli.StringFlag{
- Name: "web.telemetry-path",
- Value: "/metrics",
- Usage: "Path under which to expose metrics.",
+ &cli.StringFlag{
+ Name: "web.telemetry-path",
+ Value: "/metrics",
+ Usage: "Path under which to expose metrics.",
+ Destination: &appPath,
+ Action: func(cCtx *cli.Context, v string) error {
+ if !strings.HasPrefix(v, "/") {
+ return cli.Exit("Missing / to prefix the path: --web.telemetry-path", 2)
+ }
+ return nil
+ },
  },
- cli.BoolFlag{
+ &cli.BoolFlag{
  Name: "web.include-exporter-metrics",
  Usage: "Include metrics about the exporter itself (promhttp_*, process_*, go_*).",
+ Value: false,
  },
- cli.BoolFlag{
+ &cli.BoolFlag{
  Name: "log.debug",
  Usage: "Enable more output in the logs, otherwise INFO.",
+ Value: false,
  },
- cli.StringFlag{
+ &cli.StringFlag{
  Name: "log.output",
  Value: "stdout",
  Usage: "Destination of our logs: stdout, stderr",
+ Action: func(cCtx *cli.Context, v string) error {
+ if v != "stdout" && v != "stderr" {
+ return cli.Exit("We currently support only stdout and stderr: --log.output", 2)
+ }
+ return nil
+ },
  },
  }
 
  return DNSBLApp{
- App: app,
+ App: a,
  }
 }
 
-func (app *DNSBLApp) Bootstrap() {
- app.App.Action = func(ctx *cli.Context) error {
+func (a *DNSBLApp) Bootstrap() {
+ a.App.Action = func(ctx *cli.Context) error {
  // setup logging
+ fmt.Println("VERSION: " + appVersion)
+ handler := &slog.HandlerOptions{}
+ var writer io.Writer
+
+ if ctx.Bool("log.debug") {
+ handler.Level = slog.LevelDebug
+ }
+
  switch ctx.String("log.output") {
  case "stdout":
- log.SetOutput(os.Stdout)
+ writer = os.Stdout
  case "stderr":
- log.SetOutput(os.Stderr)
- default:
- cli.ShowAppHelp(ctx)
- return cli.NewExitError("We currently support only stdout and stderr: --log.output", 2)
+ writer = os.Stderr
  }
- if ctx.Bool("log.debug") {
- log.SetLevel(log.DebugLevel)
+
+ log := slog.New(handler.NewTextHandler(writer))
+
+ c := config.Config{
+ Logger: log.With("area", "config"),
  }
 
- cfgRbls, err := config.LoadFile(ctx.String("config.rbls"), "rbl")
+ cfgRbls, err := c.LoadFile(ctx.String("config.rbls"))
  if err != nil {
  return err
  }
 
- cfgTargets, err := config.LoadFile(ctx.String("config.targets"), "targets")
+ err = c.ValidateConfig(cfgRbls, "rbl")
+ if err != nil {
+ return fmt.Errorf("unable to load the rbls from the config: %w", err)
+ }
+
+ cfgTargets, err := c.LoadFile(ctx.String("config.targets"))
  if err != nil {
  return err
  }
 
- http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
- w.Write([]byte(`<html>
- <head><title>` + app.App.Name + `</title></head>
- <body>
- <h1>` + app.App.Name + ` @ ` + app.App.Version + `</h1>
- <p><a href="` + ctx.String("web.telemetry-path") + `">Metrics</a></p>
- <p><a href="https://github.com/Luzilla/dnsbl_exporter">Code on Github</a></p>
- </body>
- </html>`))
- })
+ err = c.ValidateConfig(cfgTargets, "targets")
+ if err != nil {
+ if !errors.Is(err, config.ErrNoServerEntries) && !errors.Is(err, config.ErrNoSuchSection) {
+ return err
+ }
+ log.Info("starting exporter without targets — check the /prober endpoint or correct the .ini file")
+ }
+
+ iHandler := index.IndexHandler{
+ Name: appName,
+ Version: appVersion,
+ Path: appPath,
+ }
+
+ http.HandleFunc("/", iHandler.Handler)
 
- rbls := config.GetRbls(cfgRbls)
- targets := config.GetTargets(cfgTargets)
+ rbls := c.GetRbls(cfgRbls)
+ targets := c.GetTargets(cfgTargets)
 
- registry := createRegistry()
+ registry := setup.CreateRegistry()
 
- collector := createCollector(rbls, targets, ctx.String("config.dns-resolver"))
- registry.MustRegister(collector)
+ rblCollector := setup.CreateCollector(rbls, targets, resolver, log.With("area", "metrics"))
+ registry.MustRegister(rblCollector)
 
- registryExporter := createRegistry()
+ registryExporter := setup.CreateRegistry()
 
  if ctx.Bool("web.include-exporter-metrics") {
- log.Infoln("Exposing exporter metrics")
+ log.Info("Exposing exporter metrics")
 
  registryExporter.MustRegister(
  prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
  prometheus.NewGoCollector(),
  )
  }
 
- handler := promhttp.HandlerFor(
- prometheus.Gatherers{
- registry,
- registryExporter,
- },
- promhttp.HandlerOpts{
- ErrorHandling: promhttp.ContinueOnError,
- Registry: registry,
- },
- )
+ mHandler := metrics.MetricsHandler{
+ Registry: registry,
+ RegistryExporter: registryExporter,
+ }
 
- http.Handle(ctx.String("web.telemetry-path"), handler)
+ http.Handle(ctx.String("web.telemetry-path"), mHandler.Handler())
+
+ pHandler := prober.ProberHandler{
+ Resolver: resolver,
+ Rbls: rbls,
+ Logger: log.With("area", "prober"),
+ }
+ http.Handle("/prober", pHandler)
 
- log.Infoln("Starting on: ", ctx.String("web.listen-address"))
+ log.Info("Starting on: " + ctx.String("web.listen-address"))
  err = http.ListenAndServe(ctx.String("web.listen-address"), nil)
  if err != nil {
  return err
@@ -157,14 +199,6 @@ func (app *DNSBLApp) Bootstrap() {
  }
 }
 
-func (app *DNSBLApp) Run(args []string) error {
- return app.App.Run(args)
-}
-
-func createCollector(rbls []string, targets []string, resolver string) *collector.RblCollector {
- return collector.NewRblCollector(rbls, targets, resolver)
-}
-
-func createRegistry() *prometheus.Registry {
- return prometheus.NewRegistry()
+func (a *DNSBLApp) Run(args []string) error {
+ return a.App.Run(args)
 }