Skip to content

Commit

Permalink
Merge pull request prometheus#10490 from DrAuYueng/fix-docker-sd-serv…
Browse files Browse the repository at this point in the history 
…ice-missing

[ENHANCEMENT] Docker SD: add MatchFirstNetwork for containers with multiple networks

Fixes docker sd service misssing in shared mode and deduplicate targets by network
  • Loading branch information
@bboreham
bboreham authored Jun 26, 2024
2 parents 2dd07fb + 0d25931 commit c5040c5
Show file tree
Hide file tree
Showing 6 changed files with 466 additions and 3 deletions.
1 change: 1 addition & 0 deletions config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -998,6 +998,7 @@ var expectedConf = &Config{
HostNetworkingHost: "localhost",
RefreshInterval: model.Duration(60 * time.Second),
HTTPClientConfig: config.DefaultHTTPClientConfig,
MatchFirstNetwork: true,
},
},
},
Expand Down
58 changes: 56 additions & 2 deletions discovery/moby/docker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,10 @@ import (
"strconv"
"time"

"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"
"github.com/go-kit/log"
"github.com/prometheus/client_golang/prometheus"
Expand Down Expand Up @@ -58,6 +60,7 @@ var DefaultDockerSDConfig = DockerSDConfig{
Filters: []Filter{},
HostNetworkingHost: "localhost",
HTTPClientConfig: config.DefaultHTTPClientConfig,
MatchFirstNetwork: true,
}

func init() {
Expand All @@ -73,7 +76,8 @@ type DockerSDConfig struct {
Filters []Filter `yaml:"filters"`
HostNetworkingHost string `yaml:"host_networking_host"`

RefreshInterval model.Duration `yaml:"refresh_interval"`
RefreshInterval model.Duration `yaml:"refresh_interval"`
MatchFirstNetwork bool `yaml:"match_first_network"`
}

// NewDiscovererMetrics implements discovery.Config.
Expand Down Expand Up @@ -119,6 +123,7 @@ type DockerDiscovery struct {
port int
hostNetworkingHost string
filters filters.Args
matchFirstNetwork bool
}

// NewDockerDiscovery returns a new DockerDiscovery which periodically refreshes its targets.
Expand All @@ -131,6 +136,7 @@ func NewDockerDiscovery(conf *DockerSDConfig, logger log.Logger, metrics discove
d := &DockerDiscovery{
port: conf.Port,
hostNetworkingHost: conf.HostNetworkingHost,
matchFirstNetwork: conf.MatchFirstNetwork,
}

hostURL, err := url.Parse(conf.Host)
Expand Down Expand Up @@ -202,6 +208,11 @@ func (d *DockerDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er
return nil, fmt.Errorf("error while computing network labels: %w", err)
}

allContainers := make(map[string]types.Container)
for _, c := range containers {
allContainers[c.ID] = c
}

for _, c := range containers {
if len(c.Names) == 0 {
continue
Expand All @@ -218,7 +229,50 @@ func (d *DockerDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er
commonLabels[dockerLabelContainerLabelPrefix+ln] = v
}

for _, n := range c.NetworkSettings.Networks {
networks := c.NetworkSettings.Networks
containerNetworkMode := container.NetworkMode(c.HostConfig.NetworkMode)
if len(networks) == 0 {
// Try to lookup shared networks
for {
if containerNetworkMode.IsContainer() {
tmpContainer, exists := allContainers[containerNetworkMode.ConnectedContainer()]
if !exists {
break
}
networks = tmpContainer.NetworkSettings.Networks
containerNetworkMode = container.NetworkMode(tmpContainer.HostConfig.NetworkMode)
if len(networks) > 0 {
break
}
} else {
break
}
}
}

if d.matchFirstNetwork && len(networks) > 1 {
// Match user defined network
if containerNetworkMode.IsUserDefined() {
networkMode := string(containerNetworkMode)
networks = map[string]*network.EndpointSettings{networkMode: networks[networkMode]}
} else {
// Get first network if container network mode has "none" value.
// This case appears under certain condition:
// 1. Container created with network set to "--net=none".
// 2. Disconnect network "none".
// 3. Reconnect network with user defined networks.
var first string
for k, n := range networks {
if n != nil {
first = k
break
}
}
networks = map[string]*network.EndpointSettings{first: networks[first]}
}
}

for _, n := range networks {
var added bool

for _, p := range c.Ports {
Expand Down
253 changes: 252 additions & 1 deletion discovery/moby/docker_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package moby
import (
"context"
"fmt"
"sort"
"testing"

"github.com/go-kit/log"
Expand Down Expand Up @@ -59,7 +60,7 @@ host: %s
tg := tgs[0]
require.NotNil(t, tg)
require.NotNil(t, tg.Targets)
require.Len(t, tg.Targets, 3)
require.Len(t, tg.Targets, 6)

for i, lbls := range []model.LabelSet{
{
Expand Down Expand Up @@ -113,9 +114,259 @@ host: %s
"__meta_docker_container_network_mode": "host",
"__meta_docker_network_ip": "",
},
{
"__address__": "172.20.0.2:3306",
"__meta_docker_container_id": "f9ade4b83199d6f83020b7c0bfd1e8281b19dbf9e6cef2cf89bc45c8f8d20fe8",
"__meta_docker_container_label_com_docker_compose_project": "dockersd",
"__meta_docker_container_label_com_docker_compose_service": "mysql",
"__meta_docker_container_label_com_docker_compose_version": "2.2.2",
"__meta_docker_container_name": "/dockersd_mysql",
"__meta_docker_container_network_mode": "dockersd_private",
"__meta_docker_network_id": "e804771e55254a360fdb70dfdd78d3610fdde231b14ef2f837a00ac1eeb9e601",
"__meta_docker_network_ingress": "false",
"__meta_docker_network_internal": "false",
"__meta_docker_network_ip": "172.20.0.2",
"__meta_docker_network_name": "dockersd_private",
"__meta_docker_network_scope": "local",
"__meta_docker_port_private": "3306",
},
{
"__address__": "172.20.0.2:33060",
"__meta_docker_container_id": "f9ade4b83199d6f83020b7c0bfd1e8281b19dbf9e6cef2cf89bc45c8f8d20fe8",
"__meta_docker_container_label_com_docker_compose_project": "dockersd",
"__meta_docker_container_label_com_docker_compose_service": "mysql",
"__meta_docker_container_label_com_docker_compose_version": "2.2.2",
"__meta_docker_container_name": "/dockersd_mysql",
"__meta_docker_container_network_mode": "dockersd_private",
"__meta_docker_network_id": "e804771e55254a360fdb70dfdd78d3610fdde231b14ef2f837a00ac1eeb9e601",
"__meta_docker_network_ingress": "false",
"__meta_docker_network_internal": "false",
"__meta_docker_network_ip": "172.20.0.2",
"__meta_docker_network_name": "dockersd_private",
"__meta_docker_network_scope": "local",
"__meta_docker_port_private": "33060",
},
{
"__address__": "172.20.0.2:9104",
"__meta_docker_container_id": "59bf76e8816af98856b90dd619c91027145ca501043b1c51756d03b085882e06",
"__meta_docker_container_label_com_docker_compose_project": "dockersd",
"__meta_docker_container_label_com_docker_compose_service": "mysqlexporter",
"__meta_docker_container_label_com_docker_compose_version": "2.2.2",
"__meta_docker_container_label_maintainer": "The Prometheus Authors <prometheus-developers@googlegroups.com>",
"__meta_docker_container_name": "/dockersd_mysql_exporter",
"__meta_docker_container_network_mode": "container:f9ade4b83199d6f83020b7c0bfd1e8281b19dbf9e6cef2cf89bc45c8f8d20fe8",
"__meta_docker_network_id": "e804771e55254a360fdb70dfdd78d3610fdde231b14ef2f837a00ac1eeb9e601",
"__meta_docker_network_ingress": "false",
"__meta_docker_network_internal": "false",
"__meta_docker_network_ip": "172.20.0.2",
"__meta_docker_network_name": "dockersd_private",
"__meta_docker_network_scope": "local",
"__meta_docker_port_private": "9104",
},
} {
t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) {
require.Equal(t, lbls, tg.Targets[i])
})
}
}

func TestDockerSDRefreshMatchAllNetworks(t *testing.T) {
sdmock := NewSDMock(t, "dockerprom")
sdmock.Setup()

e := sdmock.Endpoint()
url := e[:len(e)-1]
cfgString := fmt.Sprintf(`
---
host: %s
`, url)
var cfg DockerSDConfig
require.NoError(t, yaml.Unmarshal([]byte(cfgString), &cfg))

cfg.MatchFirstNetwork = false
reg := prometheus.NewRegistry()
refreshMetrics := discovery.NewRefreshMetrics(reg)
metrics := cfg.NewDiscovererMetrics(reg, refreshMetrics)
require.NoError(t, metrics.Register())
defer metrics.Unregister()
defer refreshMetrics.Unregister()
d, err := NewDockerDiscovery(&cfg, log.NewNopLogger(), metrics)
require.NoError(t, err)

ctx := context.Background()
tgs, err := d.refresh(ctx)
require.NoError(t, err)

require.Len(t, tgs, 1)

tg := tgs[0]
require.NotNil(t, tg)
require.NotNil(t, tg.Targets)
require.Len(t, tg.Targets, 9)

sortFunc := func(labelSets []model.LabelSet) {
sort.Slice(labelSets, func(i, j int) bool {
return labelSets[i]["__address__"] < labelSets[j]["__address__"]
})
}
expected := []model.LabelSet{
{
"__address__": "172.19.0.2:9100",
"__meta_docker_container_id": "c301b928faceb1a18fe379f6bc178727ef920bb30b0f9b8592b32b36255a0eca",
"__meta_docker_container_label_com_docker_compose_project": "dockersd",
"__meta_docker_container_label_com_docker_compose_service": "node",
"__meta_docker_container_label_com_docker_compose_version": "1.25.0",
"__meta_docker_container_label_maintainer": "The Prometheus Authors <prometheus-developers@googlegroups.com>",
"__meta_docker_container_label_prometheus_job": "node",
"__meta_docker_container_name": "/dockersd_node_1",
"__meta_docker_container_network_mode": "dockersd_default",
"__meta_docker_network_id": "7189986ab399e144e52a71b7451b4e04e2158c044b4cd2f3ae26fc3a285d3798",
"__meta_docker_network_ingress": "false",
"__meta_docker_network_internal": "false",
"__meta_docker_network_ip": "172.19.0.2",
"__meta_docker_network_label_com_docker_compose_network": "default",
"__meta_docker_network_label_com_docker_compose_project": "dockersd",
"__meta_docker_network_label_com_docker_compose_version": "1.25.0",
"__meta_docker_network_name": "dockersd_default",
"__meta_docker_network_scope": "local",
"__meta_docker_port_private": "9100",
},
{
"__address__": "172.19.0.3:80",
"__meta_docker_container_id": "c301b928faceb1a18fe379f6bc178727ef920bb30b0f9b8592b32b36255a0eca",
"__meta_docker_container_label_com_docker_compose_project": "dockersd",
"__meta_docker_container_label_com_docker_compose_service": "noport",
"__meta_docker_container_label_com_docker_compose_version": "1.25.0",
"__meta_docker_container_label_maintainer": "The Prometheus Authors <prometheus-developers@googlegroups.com>",
"__meta_docker_container_label_prometheus_job": "noport",
"__meta_docker_container_name": "/dockersd_noport_1",
"__meta_docker_container_network_mode": "dockersd_default",
"__meta_docker_network_id": "7189986ab399e144e52a71b7451b4e04e2158c044b4cd2f3ae26fc3a285d3798",
"__meta_docker_network_ingress": "false",
"__meta_docker_network_internal": "false",
"__meta_docker_network_ip": "172.19.0.3",
"__meta_docker_network_label_com_docker_compose_network": "default",
"__meta_docker_network_label_com_docker_compose_project": "dockersd",
"__meta_docker_network_label_com_docker_compose_version": "1.25.0",
"__meta_docker_network_name": "dockersd_default",
"__meta_docker_network_scope": "local",
},
{
"__address__": "localhost",
"__meta_docker_container_id": "54ed6cc5c0988260436cb0e739b7b6c9cad6c439a93b4c4fdbe9753e1c94b189",
"__meta_docker_container_label_com_docker_compose_project": "dockersd",
"__meta_docker_container_label_com_docker_compose_service": "host_networking",
"__meta_docker_container_label_com_docker_compose_version": "1.25.0",
"__meta_docker_container_name": "/dockersd_host_networking_1",
"__meta_docker_container_network_mode": "host",
"__meta_docker_network_ip": "",
},
{
"__address__": "172.20.0.2:3306",
"__meta_docker_container_id": "f9ade4b83199d6f83020b7c0bfd1e8281b19dbf9e6cef2cf89bc45c8f8d20fe8",
"__meta_docker_container_label_com_docker_compose_project": "dockersd",
"__meta_docker_container_label_com_docker_compose_service": "mysql",
"__meta_docker_container_label_com_docker_compose_version": "2.2.2",
"__meta_docker_container_name": "/dockersd_mysql",
"__meta_docker_container_network_mode": "dockersd_private",
"__meta_docker_network_id": "e804771e55254a360fdb70dfdd78d3610fdde231b14ef2f837a00ac1eeb9e601",
"__meta_docker_network_ingress": "false",
"__meta_docker_network_internal": "false",
"__meta_docker_network_ip": "172.20.0.2",
"__meta_docker_network_name": "dockersd_private",
"__meta_docker_network_scope": "local",
"__meta_docker_port_private": "3306",
},
{
"__address__": "172.20.0.2:33060",
"__meta_docker_container_id": "f9ade4b83199d6f83020b7c0bfd1e8281b19dbf9e6cef2cf89bc45c8f8d20fe8",
"__meta_docker_container_label_com_docker_compose_project": "dockersd",
"__meta_docker_container_label_com_docker_compose_service": "mysql",
"__meta_docker_container_label_com_docker_compose_version": "2.2.2",
"__meta_docker_container_name": "/dockersd_mysql",
"__meta_docker_container_network_mode": "dockersd_private",
"__meta_docker_network_id": "e804771e55254a360fdb70dfdd78d3610fdde231b14ef2f837a00ac1eeb9e601",
"__meta_docker_network_ingress": "false",
"__meta_docker_network_internal": "false",
"__meta_docker_network_ip": "172.20.0.2",
"__meta_docker_network_name": "dockersd_private",
"__meta_docker_network_scope": "local",
"__meta_docker_port_private": "33060",
},
{
"__address__": "172.21.0.2:3306",
"__meta_docker_container_id": "f9ade4b83199d6f83020b7c0bfd1e8281b19dbf9e6cef2cf89bc45c8f8d20fe8",
"__meta_docker_container_label_com_docker_compose_project": "dockersd",
"__meta_docker_container_label_com_docker_compose_service": "mysql",
"__meta_docker_container_label_com_docker_compose_version": "2.2.2",
"__meta_docker_container_name": "/dockersd_mysql",
"__meta_docker_container_network_mode": "dockersd_private",
"__meta_docker_network_id": "bfcf66a6b64f7d518f009e34290dc3f3c66a08164257ad1afc3bd31d75f656e8",
"__meta_docker_network_ingress": "false",
"__meta_docker_network_internal": "false",
"__meta_docker_network_ip": "172.21.0.2",
"__meta_docker_network_name": "dockersd_private1",
"__meta_docker_network_scope": "local",
"__meta_docker_port_private": "3306",
},
{
"__address__": "172.21.0.2:33060",
"__meta_docker_container_id": "f9ade4b83199d6f83020b7c0bfd1e8281b19dbf9e6cef2cf89bc45c8f8d20fe8",
"__meta_docker_container_label_com_docker_compose_project": "dockersd",
"__meta_docker_container_label_com_docker_compose_service": "mysql",
"__meta_docker_container_label_com_docker_compose_version": "2.2.2",
"__meta_docker_container_name": "/dockersd_mysql",
"__meta_docker_container_network_mode": "dockersd_private",
"__meta_docker_network_id": "bfcf66a6b64f7d518f009e34290dc3f3c66a08164257ad1afc3bd31d75f656e8",
"__meta_docker_network_ingress": "false",
"__meta_docker_network_internal": "false",
"__meta_docker_network_ip": "172.21.0.2",
"__meta_docker_network_name": "dockersd_private1",
"__meta_docker_network_scope": "local",
"__meta_docker_port_private": "33060",
},
{
"__address__": "172.21.0.2:9104",
"__meta_docker_container_id": "59bf76e8816af98856b90dd619c91027145ca501043b1c51756d03b085882e06",
"__meta_docker_container_label_com_docker_compose_project": "dockersd",
"__meta_docker_container_label_com_docker_compose_service": "mysqlexporter",
"__meta_docker_container_label_com_docker_compose_version": "2.2.2",
"__meta_docker_container_label_maintainer": "The Prometheus Authors <prometheus-developers@googlegroups.com>",
"__meta_docker_container_name": "/dockersd_mysql_exporter",
"__meta_docker_container_network_mode": "container:f9ade4b83199d6f83020b7c0bfd1e8281b19dbf9e6cef2cf89bc45c8f8d20fe8",
"__meta_docker_network_id": "bfcf66a6b64f7d518f009e34290dc3f3c66a08164257ad1afc3bd31d75f656e8",
"__meta_docker_network_ingress": "false",
"__meta_docker_network_internal": "false",
"__meta_docker_network_ip": "172.21.0.2",
"__meta_docker_network_name": "dockersd_private1",
"__meta_docker_network_scope": "local",
"__meta_docker_port_private": "9104",
},
{
"__address__": "172.20.0.2:9104",
"__meta_docker_container_id": "59bf76e8816af98856b90dd619c91027145ca501043b1c51756d03b085882e06",
"__meta_docker_container_label_com_docker_compose_project": "dockersd",
"__meta_docker_container_label_com_docker_compose_service": "mysqlexporter",
"__meta_docker_container_label_com_docker_compose_version": "2.2.2",
"__meta_docker_container_label_maintainer": "The Prometheus Authors <prometheus-developers@googlegroups.com>",
"__meta_docker_container_name": "/dockersd_mysql_exporter",
"__meta_docker_container_network_mode": "container:f9ade4b83199d6f83020b7c0bfd1e8281b19dbf9e6cef2cf89bc45c8f8d20fe8",
"__meta_docker_network_id": "e804771e55254a360fdb70dfdd78d3610fdde231b14ef2f837a00ac1eeb9e601",
"__meta_docker_network_ingress": "false",
"__meta_docker_network_internal": "false",
"__meta_docker_network_ip": "172.20.0.2",
"__meta_docker_network_name": "dockersd_private",
"__meta_docker_network_scope": "local",
"__meta_docker_port_private": "9104",
},
}

sortFunc(expected)
sortFunc(tg.Targets)

for i, lbls := range expected {
t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) {
require.Equal(t, lbls, tg.Targets[i])
})
}
}
Loading

0 comments on commit c5040c5

Please sign in to comment.