Consul Web-UI only shows health checks for one instance.

[apollo13]

Overview of the Issue

I have a nomad service deployed with a connect sidecar but the instances in the consul webui show different health checks (or rather miss some)

Reproduction Steps

Deploy this nomad job (Not sure how I'd create all this manually):

job "hello" {
  datacenters = ["dc1"]
  namespace = "default"

  group "hello" {
    count = 2
    network {
      mode = "bridge"
    }

    service {
      name = "hello"
      port = 8000
      tags = [
	  "traefik.enable=true",
	  "traefik.http.routers.hello.rule=PathPrefix(`/hello`)"
      ]

      connect {
        sidecar_service {
		#tags = ["dummy"]
	}
      }
    }

    task "hello" {
      driver = "docker"

      config {
        image = "hashicorp/http-echo:0.2.3"
        args = ["-listen=:8000","-text=hello"]

      }
    }

  }
}

go into the webinterface; the second instance for the job hello will only show the serf check. But, querying consul directly seems to return the correct info:

[
  {
    "Node": {
      "ID": "a6deb26b-25d8-2df7-3eec-68e383e05e23",
      "Node": "nomad01",
      "Address": "172.22.3.201",
      "Datacenter": "dc1",
      "TaggedAddresses": {
        "lan": "172.22.3.201",
        "lan_ipv4": "172.22.3.201",
        "wan": "172.22.3.201",
        "wan_ipv4": "172.22.3.201"
      },
      "Meta": {
        "consul-network-segment": ""
      },
      "CreateIndex": 6618488,
      "ModifyIndex": 6618491
    },
    "Service": {
      "Kind": "connect-proxy",
      "ID": "_nomad-task-fbebe1f5-af92-b03c-56f9-bc5ce292dff4-group-hello-hello-8000-sidecar-proxy",
      "Service": "hello-sidecar-proxy",
      "Tags": [
        "traefik.enable=true",
        "traefik.http.routers.hello.rule=PathPrefix(`/hello`)"
      ],
      "Address": "172.22.3.201",
      "TaggedAddresses": {
        "lan_ipv4": {
          "Address": "172.22.3.201",
          "Port": 25099
        },
        "wan_ipv4": {
          "Address": "172.22.3.201",
          "Port": 25099
        }
      },
      "Meta": {
        "external-source": "nomad"
      },
      "Port": 25099,
      "Weights": {
        "Passing": 1,
        "Warning": 1
      },
      "EnableTagOverride": false,
      "Proxy": {
        "DestinationServiceName": "hello",
        "DestinationServiceID": "_nomad-task-fbebe1f5-af92-b03c-56f9-bc5ce292dff4-group-hello-hello-8000",
        "LocalServiceAddress": "127.0.0.1",
        "LocalServicePort": 8000,
        "Config": {
          "bind_address": "0.0.0.0",
          "bind_port": 25099
        },
        "MeshGateway": {},
        "Expose": {}
      },
      "Connect": {},
      "CreateIndex": 6618922,
      "ModifyIndex": 6618922
    },
    "Checks": [
      {
        "Node": "nomad01",
        "CheckID": "serfHealth",
        "Name": "Serf Health Status",
        "Status": "passing",
        "Notes": "",
        "Output": "Agent alive and reachable",
        "ServiceID": "",
        "ServiceName": "",
        "ServiceTags": [],
        "Type": "",
        "Definition": {},
        "CreateIndex": 6618488,
        "ModifyIndex": 6618488
      },
      {
        "Node": "nomad01",
        "CheckID": "service:_nomad-task-fbebe1f5-af92-b03c-56f9-bc5ce292dff4-group-hello-hello-8000-sidecar-proxy:1",
        "Name": "Connect Sidecar Listening",
        "Status": "passing",
        "Notes": "",
        "Output": "TCP connect 127.0.0.1:25099: Success",
        "ServiceID": "_nomad-task-fbebe1f5-af92-b03c-56f9-bc5ce292dff4-group-hello-hello-8000-sidecar-proxy",
        "ServiceName": "hello-sidecar-proxy",
        "ServiceTags": [
          "traefik.enable=true",
          "traefik.http.routers.hello.rule=PathPrefix(`/hello`)"
        ],
        "Type": "tcp",
        "Definition": {},
        "CreateIndex": 6618922,
        "ModifyIndex": 6618924
      },
      {
        "Node": "nomad01",
        "CheckID": "service:_nomad-task-fbebe1f5-af92-b03c-56f9-bc5ce292dff4-group-hello-hello-8000-sidecar-proxy:2",
        "Name": "Connect Sidecar Aliasing _nomad-task-fbebe1f5-af92-b03c-56f9-bc5ce292dff4-group-hello-hello-8000",
        "Status": "passing",
        "Notes": "",
        "Output": "No checks found.",
        "ServiceID": "_nomad-task-fbebe1f5-af92-b03c-56f9-bc5ce292dff4-group-hello-hello-8000-sidecar-proxy",
        "ServiceName": "hello-sidecar-proxy",
        "ServiceTags": [
          "traefik.enable=true",
          "traefik.http.routers.hello.rule=PathPrefix(`/hello`)"
        ],
        "Type": "alias",
        "Definition": {},
        "CreateIndex": 6618922,
        "ModifyIndex": 6618922
      }
    ]
  },
  {
    "Node": {
      "ID": "ca76982d-28d8-346b-9c34-5d50a9c93f73",
      "Node": "nomad03",
      "Address": "172.22.3.203",
      "Datacenter": "dc1",
      "TaggedAddresses": {
        "lan": "172.22.3.203",
        "lan_ipv4": "172.22.3.203",
        "wan": "172.22.3.203",
        "wan_ipv4": "172.22.3.203"
      },
      "Meta": {
        "consul-network-segment": ""
      },
      "CreateIndex": 6618581,
      "ModifyIndex": 6618582
    },
    "Service": {
      "Kind": "connect-proxy",
      "ID": "_nomad-task-07b07a0c-8060-b854-a95c-692ce57867c0-group-hello-hello-8000-sidecar-proxy",
      "Service": "hello-sidecar-proxy",
      "Tags": [
        "traefik.enable=true",
        "traefik.http.routers.hello.rule=PathPrefix(`/hello`)"
      ],
      "Address": "172.22.3.203",
      "TaggedAddresses": {
        "lan_ipv4": {
          "Address": "172.22.3.203",
          "Port": 24773
        },
        "wan_ipv4": {
          "Address": "172.22.3.203",
          "Port": 24773
        }
      },
      "Meta": {
        "external-source": "nomad"
      },
      "Port": 24773,
      "Weights": {
        "Passing": 1,
        "Warning": 1
      },
      "EnableTagOverride": false,
      "Proxy": {
        "DestinationServiceName": "hello",
        "DestinationServiceID": "_nomad-task-07b07a0c-8060-b854-a95c-692ce57867c0-group-hello-hello-8000",
        "LocalServiceAddress": "127.0.0.1",
        "LocalServicePort": 8000,
        "Config": {
          "bind_address": "0.0.0.0",
          "bind_port": 24773
        },
        "MeshGateway": {},
        "Expose": {}
      },
      "Connect": {},
      "CreateIndex": 6618932,
      "ModifyIndex": 6618932
    },
    "Checks": [
      {
        "Node": "nomad03",
        "CheckID": "serfHealth",
        "Name": "Serf Health Status",
        "Status": "passing",
        "Notes": "",
        "Output": "Agent alive and reachable",
        "ServiceID": "",
        "ServiceName": "",
        "ServiceTags": [],
        "Type": "",
        "Definition": {},
        "CreateIndex": 6618581,
        "ModifyIndex": 6618581
      },
      {
        "Node": "nomad03",
        "CheckID": "service:_nomad-task-07b07a0c-8060-b854-a95c-692ce57867c0-group-hello-hello-8000-sidecar-proxy:1",
        "Name": "Connect Sidecar Listening",
        "Status": "passing",
        "Notes": "",
        "Output": "TCP connect 127.0.0.1:24773: Success",
        "ServiceID": "_nomad-task-07b07a0c-8060-b854-a95c-692ce57867c0-group-hello-hello-8000-sidecar-proxy",
        "ServiceName": "hello-sidecar-proxy",
        "ServiceTags": [
          "traefik.enable=true",
          "traefik.http.routers.hello.rule=PathPrefix(`/hello`)"
        ],
        "Type": "tcp",
        "Definition": {},
        "CreateIndex": 6618932,
        "ModifyIndex": 6618934
      },
      {
        "Node": "nomad03",
        "CheckID": "service:_nomad-task-07b07a0c-8060-b854-a95c-692ce57867c0-group-hello-hello-8000-sidecar-proxy:2",
        "Name": "Connect Sidecar Aliasing _nomad-task-07b07a0c-8060-b854-a95c-692ce57867c0-group-hello-hello-8000",
        "Status": "passing",
        "Notes": "",
        "Output": "No checks found.",
        "ServiceID": "_nomad-task-07b07a0c-8060-b854-a95c-692ce57867c0-group-hello-hello-8000-sidecar-proxy",
        "ServiceName": "hello-sidecar-proxy",
        "ServiceTags": [
          "traefik.enable=true",
          "traefik.http.routers.hello.rule=PathPrefix(`/hello`)"
        ],
        "Type": "alias",
        "Definition": {},
        "CreateIndex": 6618932,
        "ModifyIndex": 6618932
      }
    ]
  }
]

As you can see there are three checks registered on each node, yet the webiface shows:

I am not sure how to debug this further; I also often get "Warning an error occured while loading data" -- where can I get more details about that?

Consul info for both Client and Server

Client/Server info

agent:
	check_monitors = 0
	check_ttls = 0
	checks = 8
	services = 8
build:
	prerelease = 
	revision = 6530cf37
	version = 1.9.2
consul:
	acl = enabled
	bootstrap = false
	known_datacenters = 1
	leader = true
	leader_addr = 172.22.3.201:8300
	server = true
raft:
	applied_index = 6619061
	commit_index = 6619061
	fsm_pending = 0
	last_contact = 0
	last_log_index = 6619061
	last_log_term = 78
	last_snapshot_index = 6615104
	last_snapshot_term = 73
	latest_configuration = [{Suffrage:Voter ID:a6deb26b-25d8-2df7-3eec-68e383e05e23 Address:172.22.3.201:8300} {Suffrage:Voter ID:a0ad5ade-4ac1-37c6-f382-dbb7d10650be Address:172.22.3.202:8300} {Suffrage:Voter ID:ca76982d-28d8-346b-9c34-5d50a9c93f73 Address:172.22.3.203:8300}]
	latest_configuration_index = 0
	num_peers = 2
	protocol_version = 3
	protocol_version_max = 3
	protocol_version_min = 0
	snapshot_version_max = 1
	snapshot_version_min = 0
	state = Leader
	term = 78
runtime:
	arch = amd64
	cpu_count = 4
	goroutines = 204
	max_procs = 4
	os = linux
	version = go1.15.6
serf_lan:
	coordinate_resets = 0
	encrypted = true
	event_queue = 0
	event_time = 49
	failed = 0
	health_score = 0
	intent_queue = 0
	left = 0
	member_time = 431
	members = 3
	query_queue = 0
	query_time = 1
serf_wan:
	coordinate_resets = 0
	encrypted = true
	event_queue = 0
	event_time = 1
	failed = 0
	health_score = 0
	intent_queue = 0
	left = 0
	member_time = 221
	members = 3
	query_queue = 0
	query_time = 1

Operating system and Environment details

Standard Debian 10 on x86_64

Log Fragments

Nothing useful in the logs (not even on trace), the server shows the requests but no indication of any error

[johncowen]

Hi @apollo13

Thanks for submitting this, the HTTP response you included here was super helpful, so thanks for including that!

Sorry to ask, but just incase - do any filters checked in the Type dropdown in the filter bar?

[apollo13]

Sorry to ask, but just incase - do any filters checked in the Type dropdown in the filter bar?

Nope, Nothing checked. FWIW if I go back to the other instance (not consistently but still somewhat reproducable) I see a list of more than 3 checks (currently 30 or so). Mostly I just see the checks that should be there on nomad1 though.

Neither network console nor js console shows any interesting errors.

[apollo13]

I can paste the responses from the webui as the browser sees them as needed. Or more consul API calls if that helps you. Just tell me what you need.

[johncowen]

Hi @apollo13

Thanks again for all the info you provided here! It really helped to get a reproduction up and running. We have a fix in the works, so hopefully it will be in the next release.

👍

[apollo13]

Lovely thanks, this issue made debugging connect issues rather hard till I realized that those are not my issues :)