metrics from gateway endpoints

Dockerfile

@@ -0,0 +1,13 @@
+FROM hitachi.jfrog.io/docker/golang:1.21 as builder
+ARG OTEL_VERSION=0.106.1
+WORKDIR /
+COPY . .
+RUN go install go.opentelemetry.io/collector/cmd/builder@v${OTEL_VERSION}
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 builder --config builder-config.yaml
+RUN mkdir -p /oteldata
+
+FROM scratch
+COPY --from=builder /otelcol-dev/otelcol-dev /otel
+COPY --from=builder --chown=10001:10001 /oteldata /oteldata
+USER 10001
+ENTRYPOINT ["/otel"]

builder-config.yaml

@@ -0,0 +1,25 @@
+dist:
+  name: otelcol-dev
+  description: Basic OTel Collector distribution for Developers
+  output_path: ./otelcol-dev
+  otelcol_version: 0.106.1
+
+exporters:
+  - gomod:
+      go.opentelemetry.io/collector/exporter/debugexporter v0.106.1
+  - gomod:
+      go.opentelemetry.io/collector/exporter/otlpexporter v0.106.1
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.106.1
+
+processors:
+  - gomod:
+      go.opentelemetry.io/collector/processor/batchprocessor v0.106.1
+
+receivers:
+  - gomod:
+      go.opentelemetry.io/collector/receiver/otlpreceiver v0.106.1
+  - gomod:
+      github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.106.1
+  - gomod:
+      github.com/adarshsingh2/custom-collector/ucpmetricreceiver v0.0.1
+

config.yaml

@@ -0,0 +1,146 @@
+receivers:
+  metricsreceiver: # this line represents the ID of your receiver
+    scrape_config:
+      - interval: 40s
+        device_type: "storage"
+        timeout: 88s
+        metric_keys:
+          - "portIopsAvg"
+          - "portIopsMax"
+          - "portIopsMin"
+          - "portKbpsAvg"
+          - "portKbpsMax"
+          - "portKbpsMin"
+          - "logical_unit_ldevIO_randomIO_read"
+          - "logical_unit_ldevIO_totalIO_write"
+          - "logical_unit_random_iops"
+          - "logical_unit_write_latency"
+          - "logical_unit_total_write_hit"
+          - "logical_unit_ldevBlockIO_totalIO_read"
+          - "logical_unit_ldevIO_sequentialIO_writeHit"
+          - "logical_unit_logicalUnitIO_sequentialIO_read"
+          - "logical_unit_write_cache_hit_ratio"
+          - "logical_unit_logicalUnitBlockIO_randomIO_write"
+          - "logical_unit_logicalUnitBlockIO_sequentialIO_read"
+          - "logical_unit_ldevLatency_write"
+          - "logical_unit_total_cache_hits"
+          - "logical_unit_ldevBlockIO_sequentialIO_write"
+          - "logical_unit_logicalUnitBlockIO_randomIO_read"
+          - "logical_unit_ldevLatency_writeMax"
+          - "logical_unit_logicalUnitIO_randomIO_readHit"
+          - "logical_unit_logicalUnitIO_randomIO_writeHit"
+          - "logical_unit_logicalUnitIO_sequentialIO_readHit"
+          - "logical_unit_logicalUnitIO_totalIO_read"
+          - "logical_unit_iops"
+          - "logical_unit_ldevBlockIO_randomIO_read"
+          - "logical_unit_max_write_latency"
+          - "logical_unit_read_latency"
+          - "logical_unit_logicalUnitIO_randomIO_read"
+          - "logical_unit_read_cache_hit_ratio"
+          - "logical_unit_ldevIO_totalIO_readHit"
+          - "logical_unit_ldevBlockIO_sequentialIO_writeHit"
+          - "logical_unit_ldevBlockIO_sequentialIO_read"
+          - "logical_unit_ldevIO_totalIO_read"
+          - "logical_unit_ldevBlockIO_sequentialIO_readHit"
+          - "logical_unit_ldevIO_totalIO_writeHit"
+          - "logical_unit_logicalUnitBlockIO_randomIO_readHit"
+          - "logical_unit_logicalUnitBlockIO_totalIO_write"
+          - "logical_unit_logicalUnitIO_sequentialIO_write"
+          - "logical_unit_logicalUnitBlockIO_sequentialIO_writeHit"
+          - "logical_unit_logicalUnitBlockIO_totalIO_writeHit"
+          - "logical_unit_ldevBlockIO_randomIO_write"
+          - "logical_unit_ldevBlockIO_randomIO_writeHit"
+          - "logical_unit_ldevIO_randomIO_write"
+          - "logical_unit_ldevIO_randomIO_writeHit"
+          - "logical_unit_ldevIO_sequentialIO_read"
+          - "logical_unit_ldevIO_sequentialIO_write"
+          - "logical_unit_ldevLatency_read"
+          - "logical_unit_ldevLatency_readMax"
+          - "logical_unit_sequential_iops"
+          - "logical_unit_total_read_io"
+          - "logical_unit_total_write_io"
+          - "logical_unit_ldevBlockIO_randomIO_readHit"
+          - "logical_unit_ldevBlockIO_totalIO_readHit"
+          - "logical_unit_ldevBlockIO_totalIO_write"
+          - "logical_unit_ldevBlockIO_totalIO_writeHit"
+          - "logical_unit_ldevIO_randomIO_readHit"
+          - "logical_unit_ldevIO_sequentialIO_readHit"
+          - "logical_unit_logicalUnitBlockIO_randomIO_writeHit"
+          - "logical_unit_logicalUnitBlockIO_sequentialIO_readHit"
+          - "logical_unit_logicalUnitBlockIO_sequentialIO_write"
+          - "logical_unit_logicalUnitBlockIO_totalIO_read"
+          - "logical_unit_logicalUnitBlockIO_totalIO_readHit"
+          - "logical_unit_logicalUnitIO_randomIO_write"
+          - "logical_unit_logicalUnitIO_sequentialIO_writeHit"
+          - "logical_unit_logicalUnitIO_totalIO_readHit"
+          - "logical_unit_logicalUnitIO_totalIO_write"
+          - "logical_unit_logicalUnitIO_totalIO_writeHit"
+          - "logical_unit_max_read_latency"
+          - "logical_unit_total_read_hit"
+        timestamp_keys:
+          - "time"
+
+
+
+      - interval: 40s
+        device_type: "compute"
+        timeout: 30s
+        metric_keys:
+        - "avgPower1week"
+        - "avgPower20min"
+        - "avgPower24hour"
+        - "avgPower5min"
+        - "maxPower20min"
+        - "maxPower24hour"
+        - "maxPower5min"
+        - "maxPower1week"
+        - "minPower20min"
+        - "minPower24hour"
+        - "minPower5min"
+        - "minPower1week"
+        - "powerInputVoltage"
+        - "presentPowerReading"
+        - "sensor_temperature"
+        - "sensor_fan_value"
+        timestamp_keys:
+          - "time"
+      # #     - "timeStamp"
+    endpoint: "https://genai-qdrant.genai.sc.eng.hitachivantara.com/metrics"
+    auth_endpoint: "https://172.25.92.81/porcelain/v2/auth/login"
+    compute_endpoint: "https://172.25.92.81/porcelain/v2/systems"
+    metrics_base_url: "https://172.25.92.81"
+processors:
+  batch:
+    send_batch_size: 2000
+    timeout: 10s
+exporters:
+  prometheusremotewrite:
+    endpoint: "http://127.0.0.1:4000/v1/prometheus/write?db=public"
+    timeout: 120s
+    headers:
+      Authorization: "Bearer your-auth-token"
+    retry_on_failure:
+      enabled: true
+      initial_interval: 5s
+      max_interval: 90s
+      max_elapsed_time: 300s
+    # remote_write_queue:
+    #   enabled: true
+    #   queue_size: 20000  # Keep this large enough to handle short bursts of traffic
+    #   num_consumers: 5
+    # wal:  # Enable Write-Ahead Log (WAL) to persist metrics to disk
+    #   directory: ./prom_rw  # Directory to store WAL files
+    #   buffer_size: 1000      # Optional: Number of elements to buffer before truncating
+    #   truncate_frequency: 1m # Optional: How often to truncate the WAL (default: 1m)
+
+
+service:
+  pipelines:
+    metrics:
+      receivers: [metricsreceiver]
+      processors: [batch]
+      exporters: [prometheusremotewrite]
+
+  telemetry:
+    logs:
+      level: "debug"

ucpmetricreceiver/computemetricparser.go

@@ -0,0 +1 @@
+package metricsreceiver

ucpmetricreceiver/config.go

@@ -8,7 +8,7 @@ import (
 // ScrapeConfig represents a single scrape configuration within the receiver's config.yaml
 type ScrapeConfig struct {
 	Interval      string   `mapstructure:"interval"`
-	Device        string   `mapstructure:"device"`
+	DeviceType    string   `mapstructure:"device_type"`
 	Timeout       string   `mapstructure:"timeout"`
 	MetricKeys    []string `mapstructure:"metric_keys"`
 	TimestampKeys []string `mapstructure:"timestamp_keys"`
@@ -46,7 +46,7 @@ func (sc *ScrapeConfig) Validate() error {
 		return fmt.Errorf("the interval in scrape_config has to be set to at least 10 secs (10s)")
 	}
 
-	if sc.Device == "" {
+	if sc.DeviceType == "" {
 		return fmt.Errorf("device in scrape_config cannot be empty")
 	}