set up oci metrics in grafana

hive.nix

@@ -87,6 +87,7 @@ in
 // (mkNodePool {
   names = with builtins; fromJSON (readFile "${self}/terraform/metal/oci_control.json");
   module = {
+   # hqsw has 1 core not 2
     imports = [ ./machines/ociControlWorker srvos.nixosModules.server ];
     deployment.tags = [ "oci-control" ];
     deployment.buildOnTarget = false;

jobs/roach.nix

@@ -88,7 +88,8 @@ let
           "traefik.tcp.routers.roach-db.rule=HostSNI(`roach-db.traefik`) || HostSNI(`roach-db.tfk.nd`)"
           "traefik.tcp.routers.roach-db.entrypoints=sql"
         ];
-        checks = [{
+        checks = [
+        {
           name = "health-ready";
           path = "/health?ready=1";
           tlsSkipVerify = true;
@@ -99,7 +100,18 @@ let
           # we want nomad to ignore this, it's traefik
           # that should respect the check
           onUpdate = "ignore";
-        }];
+        }
+        {
+          name = "health";
+          path = "/health";
+          tlsSkipVerify = true;
+          type = "http";
+          portLabel = "metrics";
+          interval = 5 * lib.seconds;
+          timeout = 1 * lib.seconds;
+          onUpdate = "require_healthy";
+        }
+        ];
       }
       {
         name = "roach-web";

terraform/global_variables.tf

@@ -7,4 +7,8 @@ locals {
     project = "dcotta-com"
     region = "europe-west3"
   }
+
+  oci = {
+    tenancyOcid = "ocid1.tenancy.oc1..aaaaaaaa5umazgc4ircdizxgjixccyal2nkmemxnpzcrwgwjsrzcz4omw32q"
+  }
 }

terraform/grafana/global_variables.tf

@@ -0,0 +1 @@
+../global_variables.tf

terraform/grafana/oci.tf

@@ -1,20 +1,61 @@
 resource "oci_identity_user" "grafana" {
-  description = ""
+  description = "grafana"
   name        = "grafana"
+  email       = "grafana.sa@dcotta.com"
 }
 
 resource "oci_identity_group" "metrics" {
-  description = "grafana"
+  description = "grafana metrics"
   name        = "metrics"
-
 }
 
 resource "oci_identity_policy" "metrics" {
   compartment_id = local.ociRoot
-  description    = ""
+  description    = "grafana metrics"
   name           = "metrics"
   statements = [
-    "allow group grafana to read metrics in tenancy",
-    "allow group grafana to read compartments in tenancy",
+    "allow group metrics to read metrics in tenancy",
+    "allow group metrics to inspect metrics in tenancy",
+#     "allow group grafana to inspect metrics in compartment cottand9000",
+    "allow group metrics to read compartments in tenancy",
   ]
 }
+
+resource "oci_identity_user_group_membership" "grafana" {
+  group_id = oci_identity_group.metrics.id
+  user_id  = oci_identity_user.grafana.id
+}
+
+data "vault_kv_secret_v2" "grafana_oci" {
+  mount = "secret"
+  name  = "nomad/job/grafana/oci_user"
+}
+
+resource "oci_identity_api_key" "grafana" {
+  key_value = data.vault_kv_secret_v2.grafana_oci.data["publicKeyPem"]
+  user_id   = oci_identity_user.grafana.id
+}
+
+resource "grafana_data_source" "oci" {
+  name = "oci"
+  type = "oci-metrics-datasource"
+}
+
+
+resource "grafana_data_source_config" "oci" {
+  uid = grafana_data_source.oci.uid
+  # https://github.com/oracle/oci-grafana-metrics/blob/master/docs/datasource_configuration.md
+  json_data_encoded = jsonencode({
+    "profile0" : "DEFAULT",
+    "region0" : "eu-frankfurt-1",
+    "tenancymode" : "single",
+    "environment" : "local",
+  })
+
+  secure_json_data_encoded = jsonencode({
+    user0 : oci_identity_user.grafana.id
+    tenancy0 : local.ociTenancyOcid
+    fingerprint0 : oci_identity_api_key.grafana.fingerprint
+    privkey0 : data.vault_kv_secret_v2.grafana_oci.data["privateKey"]
+  })
+}

terraform/grafana/provider.tf

@@ -12,12 +12,24 @@ terraform {
       source  = "sebastiaan-dev/bitwarden-secrets"
       version = "0.1.2"
     }
+    vault = {
+      source  = "hashicorp/vault"
+      version = "~> 4.2.0"
+    }
   }
 }
 
 data "external" "keychain-bw-token" {
   program = ["keychain-get", "bitwarden/secret/m3-cli"]
 }
+
+provider "vault" {
+  address         = local.vault_addr
+}
+
+provider "bitwarden-secrets" {
+  access_token = data.external.keychain-bw-token.result.value
+}
 provider "grafana" {
   url = "https://grafana.tfk.nd/"
   #   auth = var.grafana_auth
@@ -30,7 +42,9 @@ data "bitwarden-secrets_secret" "ociTfPrivateKey" {
 locals {
   ociUser = jsondecode(data.bitwarden-secrets_secret.ociTfPrivateKey.value)
   ociRoot = local.ociUser["ocid"]
+  ociTenancyOcid = local.ociUser["ocid"]
 }
+
 provider "oci" {
   private_key  = local.ociUser["private_key"]
   tenancy_ocid = local.ociUser["ocid"]