Scrape job is missing a juju_unit label

[sed-i]

Bug Description

Both prometheus and alertmanager use very similar "self scrape" jobs (prom, am), but for some reason, the prometheus scrape job has a juju_unit label, while alertmanager does not (see screenshot).

To Reproduce

Deploy the following bundle:

graph LR
prom ---|metrics-endpoint| first
prom ---|metrics-endpoint| second

Loading

bundle: kubernetes
applications:
  prom:
    charm: prometheus-k8s
    channel: latest/edge
    revision: 209
    resources:
      prometheus-image: 148
    scale: 1
    trust: true
  first:
    charm: alertmanager-k8s
    channel: latest/edge
    revision: 125
    resources:
      alertmanager-image: 94
    scale: 2
    trust: true
  second:
    charm: alertmanager-k8s
    channel: latest/edge
    revision: 125
    resources:
      alertmanager-image: 94
    scale: 2
    trust: true
relations:
- - prom:metrics-endpoint
  - first:self-metrics-endpoint
- - prom:metrics-endpoint
  - second:self-metrics-endpoint

Environment

Model        Controller  Cloud/Region        Version  SLA          Timestamp
welcome-k8s  microk8s    microk8s/localhost  3.5.1    unsupported  16:41:38-04:00

App     Version  Status  Scale  Charm             Channel      Rev  Address         Exposed  Message
first   0.27.0   active      2  alertmanager-k8s  latest/edge  125  10.152.183.34   no       
prom    2.52.0   active      1  prometheus-k8s    latest/edge  209  10.152.183.223  no       
second  0.27.0   active      2  alertmanager-k8s  latest/edge  125  10.152.183.181  no       

Relevant log output

➜  ~ jssh prom/0 curl localhost:9090/api/v1/targets | jq
{
  "status": "success",
  "data": {
    "activeTargets": [
      {
        "discoveredLabels": {
          "__address__": "first-0.first-endpoints.welcome-k8s.svc.cluster.local:9093",
          "__metrics_path__": "/metrics",
          "__scheme__": "http",
          "__scrape_interval__": "1m",
          "__scrape_timeout__": "10s",
          "job": "juju_welcome-k8s_1de328c6_first_prometheus_scrape",
          "juju_application": "first",
          "juju_charm": "alertmanager-k8s",
          "juju_model": "welcome-k8s",
          "juju_model_uuid": "1de328c6-0382-4478-86f0-ae880a381f81"
        },
        "labels": {
          "instance": "welcome-k8s_1de328c6-0382-4478-86f0-ae880a381f81_first",
          "job": "juju_welcome-k8s_1de328c6_first_prometheus_scrape",
          "juju_application": "first",
          "juju_charm": "alertmanager-k8s",
          "juju_model": "welcome-k8s",
          "juju_model_uuid": "1de328c6-0382-4478-86f0-ae880a381f81"
        },
        "scrapePool": "juju_welcome-k8s_1de328c6_first_prometheus_scrape",
        "scrapeUrl": "http://first-0.first-endpoints.welcome-k8s.svc.cluster.local:9093/metrics",
        "globalUrl": "http://first-0.first-endpoints.welcome-k8s.svc.cluster.local:9093/metrics",
        "lastError": "",
        "lastScrape": "2024-07-08T20:27:34.565866864Z",
        "lastScrapeDuration": 0.004640402,
        "health": "up",
        "scrapeInterval": "1m",
        "scrapeTimeout": "10s"
      },
      {
        "discoveredLabels": {
          "__address__": "first-1.first-endpoints.welcome-k8s.svc.cluster.local:9093",
          "__metrics_path__": "/metrics",
          "__scheme__": "http",
          "__scrape_interval__": "1m",
          "__scrape_timeout__": "10s",
          "job": "juju_welcome-k8s_1de328c6_first_prometheus_scrape",
          "juju_application": "first",
          "juju_charm": "alertmanager-k8s",
          "juju_model": "welcome-k8s",
          "juju_model_uuid": "1de328c6-0382-4478-86f0-ae880a381f81"
        },
        "labels": {
          "instance": "welcome-k8s_1de328c6-0382-4478-86f0-ae880a381f81_first",
          "job": "juju_welcome-k8s_1de328c6_first_prometheus_scrape",
          "juju_application": "first",
          "juju_charm": "alertmanager-k8s",
          "juju_model": "welcome-k8s",
          "juju_model_uuid": "1de328c6-0382-4478-86f0-ae880a381f81"
        },
        "scrapePool": "juju_welcome-k8s_1de328c6_first_prometheus_scrape",
        "scrapeUrl": "http://first-1.first-endpoints.welcome-k8s.svc.cluster.local:9093/metrics",
        "globalUrl": "http://first-1.first-endpoints.welcome-k8s.svc.cluster.local:9093/metrics",
        "lastError": "",
        "lastScrape": "2024-07-08T20:28:06.257448557Z",
        "lastScrapeDuration": 0.003320719,
        "health": "up",
        "scrapeInterval": "1m",
        "scrapeTimeout": "10s"
      },
      {
        "discoveredLabels": {
          "__address__": "second-0.second-endpoints.welcome-k8s.svc.cluster.local:9093",
          "__metrics_path__": "/metrics",
          "__scheme__": "http",
          "__scrape_interval__": "1m",
          "__scrape_timeout__": "10s",
          "job": "juju_welcome-k8s_1de328c6_second_prometheus_scrape",
          "juju_application": "second",
          "juju_charm": "alertmanager-k8s",
          "juju_model": "welcome-k8s",
          "juju_model_uuid": "1de328c6-0382-4478-86f0-ae880a381f81"
        },
        "labels": {
          "instance": "welcome-k8s_1de328c6-0382-4478-86f0-ae880a381f81_second",
          "job": "juju_welcome-k8s_1de328c6_second_prometheus_scrape",
          "juju_application": "second",
          "juju_charm": "alertmanager-k8s",
          "juju_model": "welcome-k8s",
          "juju_model_uuid": "1de328c6-0382-4478-86f0-ae880a381f81"
        },
        "scrapePool": "juju_welcome-k8s_1de328c6_second_prometheus_scrape",
        "scrapeUrl": "http://second-0.second-endpoints.welcome-k8s.svc.cluster.local:9093/metrics",
        "globalUrl": "http://second-0.second-endpoints.welcome-k8s.svc.cluster.local:9093/metrics",
        "lastError": "",
        "lastScrape": "2024-07-08T20:28:15.73840897Z",
        "lastScrapeDuration": 0.002313592,
        "health": "up",
        "scrapeInterval": "1m",
        "scrapeTimeout": "10s"
      },
      {
        "discoveredLabels": {
          "__address__": "second-1.second-endpoints.welcome-k8s.svc.cluster.local:9093",
          "__metrics_path__": "/metrics",
          "__scheme__": "http",
          "__scrape_interval__": "1m",
          "__scrape_timeout__": "10s",
          "job": "juju_welcome-k8s_1de328c6_second_prometheus_scrape",
          "juju_application": "second",
          "juju_charm": "alertmanager-k8s",
          "juju_model": "welcome-k8s",
          "juju_model_uuid": "1de328c6-0382-4478-86f0-ae880a381f81"
        },
        "labels": {
          "instance": "welcome-k8s_1de328c6-0382-4478-86f0-ae880a381f81_second",
          "job": "juju_welcome-k8s_1de328c6_second_prometheus_scrape",
          "juju_application": "second",
          "juju_charm": "alertmanager-k8s",
          "juju_model": "welcome-k8s",
          "juju_model_uuid": "1de328c6-0382-4478-86f0-ae880a381f81"
        },
        "scrapePool": "juju_welcome-k8s_1de328c6_second_prometheus_scrape",
        "scrapeUrl": "http://second-1.second-endpoints.welcome-k8s.svc.cluster.local:9093/metrics",
        "globalUrl": "http://second-1.second-endpoints.welcome-k8s.svc.cluster.local:9093/metrics",
        "lastError": "",
        "lastScrape": "2024-07-08T20:28:15.377956267Z",
        "lastScrapeDuration": 0.003625857,
        "health": "up",
        "scrapeInterval": "1m",
        "scrapeTimeout": "10s"
      },
      {
        "discoveredLabels": {
          "__address__": "prom-0.prom-endpoints.welcome-k8s.svc.cluster.local:9090",
          "__metrics_path__": "/metrics",
          "__scheme__": "http",
          "__scrape_interval__": "5s",
          "__scrape_timeout__": "5s",
          "host": "localhost",
          "job": "prometheus",
          "juju_application": "prom",
          "juju_charm": "prometheus-k8s",
          "juju_model": "welcome-k8s",
          "juju_model_uuid": "1de328c6-0382-4478-86f0-ae880a381f81",
          "juju_unit": "prom/0"
        },
        "labels": {
          "host": "localhost",
          "instance": "welcome-k8s_1de328c6-0382-4478-86f0-ae880a381f81_prom_prom/0",
          "job": "prometheus",
          "juju_application": "prom",
          "juju_charm": "prometheus-k8s",
          "juju_model": "welcome-k8s",
          "juju_model_uuid": "1de328c6-0382-4478-86f0-ae880a381f81",
          "juju_unit": "prom/0"
        },
        "scrapePool": "prometheus",
        "scrapeUrl": "http://prom-0.prom-endpoints.welcome-k8s.svc.cluster.local:9090/metrics",
        "globalUrl": "http://prom-0.prom-endpoints.welcome-k8s.svc.cluster.local:9090/metrics",
        "lastError": "",
        "lastScrape": "2024-07-08T20:28:22.805860841Z",
        "lastScrapeDuration": 0.003443195,
        "health": "up",
        "scrapeInterval": "5s",
        "scrapeTimeout": "5s"
      }
    ],
    "droppedTargets": [],
    "droppedTargetCounts": {
      "juju_welcome-k8s_1de328c6_first_prometheus_scrape": 0,
      "juju_welcome-k8s_1de328c6_second_prometheus_scrape": 0,
      "prometheus": 0
    }
  }
}

Additional context

No response