Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

release/v4.1.0 #62

Merged
merged 8 commits into from
Feb 10, 2025
Merged

release/v4.1.0 #62

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
861b32e
feat: add metric to export instance age
gjasny Jan 26, 2025
e812f5e
better communication errors telling us what and why it failed
NorseGaud Jan 29, 2025
1a0b4b0
Merge pull request #60 from gjasny/add-vm-age
NorseGaud Jan 29, 2025
7eedab3
Merge pull request #61 from veertuinc/better-comm-errors
NorseGaud Jan 29, 2025
1bc49ca
bumps
NorseGaud Jan 29, 2025
a346829
using TS to handle non-started states
NorseGaud Jan 29, 2025
522c800
quick fix
NorseGaud Jan 29, 2025
2822cb9
creation time only happens on scheduling, then never changes
NorseGaud Jan 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,8 @@ anka_instance_state_count | Count of Instances in a particular State (labels: ar
anka_instance_state_per_template_count | Count of Instances in a particular state, per Template (labels: state, template_uuid, template_name)
anka_instance_state_per_group_count | Count of Instances in a particular state, per Group (labels: state, group_name)
-- | --
anka_instance_max_age_per_template_seconds | Age of oldest Instance in a particular state, per Template (labels: state, template_uuid, template_name)
-- | --
anka_node_instance_count | Count of Instances running on the Node (labels: id, name, arch)
anka_node_instance_capacity | Total Instance slots (capacity) on the Node (labels: id, name, arch)
anka_node_states | Node state (1 = current state) (labels: id, name, state)
Expand Down
2 changes: 1 addition & 1 deletion VERSION
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
4.0.1
4.1.0
18 changes: 9 additions & 9 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ toolchain go1.22.5

require (
github.com/prometheus/client_golang v1.20.5
github.com/prometheus/exporter-toolkit v0.13.1
github.com/prometheus/exporter-toolkit v0.13.2
)

require (
Expand All @@ -20,14 +20,14 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.60.1 // indirect
github.com/prometheus/common v0.61.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
golang.org/x/crypto v0.28.0 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/oauth2 v0.23.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.26.0 // indirect
golang.org/x/text v0.19.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/net v0.32.0 // indirect
golang.org/x/oauth2 v0.24.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/text v0.21.0 // indirect
google.golang.org/protobuf v1.35.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
40 changes: 20 additions & 20 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,30 +33,30 @@ github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+
github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc=
github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw=
github.com/prometheus/exporter-toolkit v0.13.1 h1:Evsh0gWQo2bdOHlnz9+0Nm7/OFfIwhE2Ws4A2jIlR04=
github.com/prometheus/exporter-toolkit v0.13.1/go.mod h1:ujdv2YIOxtdFxxqtloLpbqmxd5J0Le6IITUvIRSWjj0=
github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ=
github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s=
github.com/prometheus/exporter-toolkit v0.13.2 h1:Z02fYtbqTMy2i/f+xZ+UK5jy/bl1Ex3ndzh06T/Q9DQ=
github.com/prometheus/exporter-toolkit v0.13.2/go.mod h1:tCqnfx21q6qN1KA4U3Bfb8uWzXfijIrJz3/kTIqMV7g=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
Expand Down
32 changes: 28 additions & 4 deletions src/client/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package client

import (
"fmt"
"io"
"reflect"
"runtime"
"sync"
Expand All @@ -26,8 +27,9 @@ type Client struct {

func NewClient(addr, username, password string, interval int, certs ClientTLSCerts, uak UAK) (*Client, error) {
communicator, err := NewCommunicator(addr, username, password, certs, uak)
if err != nil {
return nil, err
if err != nil || communicator == nil {
log.Error("Failed to create communicator")
return nil, fmt.Errorf("failed to create communicator: %v", err)
}
c := &Client{
events: map[events.Event][]func(interface{}) error{
Expand All @@ -42,7 +44,26 @@ func NewClient(addr, username, password string, interval int, certs ClientTLSCer
errorTimeoutSeconds: 10,
}
if err := c.communicator.TestConnection(); err != nil {
log.Fatal(fmt.Sprintf("Error testing connection: %s", err.Error()))
response, err := c.communicator.getResponse("/api/v1/status", "", "")
if err != nil {
log.Error(fmt.Sprintf("Error getting response: %s", err.Error()))
} else {
bodyBytes := make([]byte, 1024)
for {
n, err := response.Body.Read(bodyBytes)
if n > 0 {
log.Error(fmt.Sprintf("call to %s returned %d code and body of '%s'", response.Request.URL, response.StatusCode, string(bodyBytes[:n])))
return nil, fmt.Errorf("failed to test connection")
}
if err != nil {
if err != io.EOF {
log.Error(fmt.Sprintf("Error reading response body: %s", err.Error()))
return nil, fmt.Errorf("failed to test connection")
}
break
}
}
}
return nil, err
}
return c, nil
Expand All @@ -51,7 +72,10 @@ func NewClient(addr, username, password string, interval int, certs ClientTLSCer
func (client *Client) Init() {
// We must first populate the data from the Controller API that is going to be stored in state before we attempt to create metrics from it
// Order matters here since GetVmsData for example relies on RegistryTemplatesData
client.communicator.GetRegistryTemplatesData()
_, err := client.communicator.GetRegistryTemplatesData()
if err != nil {
log.Error(fmt.Sprintf("Error getting registry templates data: %v", err))
}
go client.initDataLoop(client.communicator.GetNodesData, events.EVENT_NODE_UPDATED)
go client.initDataLoop(client.communicator.GetVmsData, events.EVENT_VM_DATA_UPDATED)
go client.initDataLoop(client.communicator.GetRegistryDiskData, events.EVENT_REGISTRY_DISK_DATA_UPDATED)
Expand Down
4 changes: 2 additions & 2 deletions src/client/communicator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ func (comm *Communicator) GetRegistryTemplatesData() (interface{}, error) {
resp := &types.RegistryTemplateResponse{}
templates, err := comm.getData(endpoint, resp)
if err != nil {
return nil, fmt.Errorf("getting registry templates error: %s", err)
return nil, fmt.Errorf("getting registry templates error: %s", err.Error())
}
templatesArray := templates.([]types.Template)
templatesMap := state.GetState().GetTemplatesMap()
Expand All @@ -163,7 +163,7 @@ func (comm *Communicator) GetRegistryTemplatesData() (interface{}, error) {
resp := &types.RegistryTemplateTagsResponse{}
tagsData, err := comm.getData(endpoint, resp)
if err != nil {
return nil, fmt.Errorf("getting registry template %s/%s tags error: %s", template.UUID, template.Name, err)
return nil, fmt.Errorf("getting registry template %s/%s tags error: %s", template.UUID, template.Name, err.Error())
}
tags := tagsData.(types.RegistryTemplateTags)
templatesArray[i].Tags = tags.Versions
Expand Down
59 changes: 59 additions & 0 deletions src/metrics/instance_state_per.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
package metrics

import (
"fmt"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/veertuinc/anka-prometheus-exporter/src/events"
"github.com/veertuinc/anka-prometheus-exporter/src/log"
"github.com/veertuinc/anka-prometheus-exporter/src/types"
)

Expand Down Expand Up @@ -150,6 +154,61 @@ var ankaInstanceStatePerMetrics = []InstanceStatePerMetric{
}
},
},
{
BaseAnkaMetric: BaseAnkaMetric{
metric: CreateGaugeMetricVec("anka_instance_max_age_per_template_seconds", "Age of oldest Instance in a particular state, per Template. Visible only for templates with at least one instance (label: state, template_uuid, template_name)", []string{"state", "template_uuid", "template_name"}),
event: events.EVENT_VM_DATA_UPDATED,
},
HandleData: func(instances []types.Instance, metric *prometheus.GaugeVec) {
var InstanceAgePerTemplateMaximumMap = map[string]map[string]int{}
var instanceTemplates []string
var instanceTemplatesMap = map[string]string{}
now := time.Now()
for _, instance := range instances {
instanceTemplates = append(instanceTemplates, instance.Vm.TemplateUUID)
instanceTemplatesMap[instance.Vm.TemplateUUID] = instance.Vm.TemplateName
}
instanceTemplates = uniqueThisStringArray(instanceTemplates)
for _, wantedState := range types.InstanceStates {
if _, ok := InstanceAgePerTemplateMaximumMap[wantedState]; !ok {
InstanceAgePerTemplateMaximumMap[wantedState] = make(map[string]int)
}
for _, wantedInstanceTemplate := range instanceTemplates {
age := 0.0
for _, instance := range instances {
if instance.Vm.State == wantedState {
if instance.Vm.TemplateUUID == wantedInstanceTemplate {
var instanceTime time.Time
var err error
if instance.Vm.State != "Started" {
instanceTime, err = time.Parse(time.RFC3339, instance.Vm.LastUpdateTime) // can't use CreationTime because it's not updated for non-started instances
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@gjasny Please review

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The idea of the age was to see how old the oldest instance (of a certain template) is. If the timestamp that is used to calculate the age is now updated as well, the "birth date" of the instance is moving which defeats the purpose. E.g. in the case of pulling, the instance is now reported with a maximum value of 5 seconds.

If the cr_time field is not present for the Scheduling state (and others) we should exclude those states (or only include states which are guaranteed to have the cr_time field). Or skip all instances that don't have the cr_time field.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On my controller even a "Scheduling" instance has the cr_time filed:

      "vm": {
        "instance_id": "050823b7-ab5c-4947-77bb-06aa2a4a69d6",
        "instance_state": "Scheduling",
        "cr_time": "2025-01-30T07:55:07.334254871Z",

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@arturmelanchyk , can you check to see which states use cr_time and which use ts?

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in the most recent version of the controller (1.45.0) all instances in all states have both cr_time and ts

Copy link
Member Author

@NorseGaud NorseGaud Jan 31, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cr_time only set on an instance creation (in the DB) and never changed
ts gets updated from time to time due to different events, like save image, termination etc
both ts and cr_time are members of an Instance object, they do not depend on the vm (regardless if the vm has started or not)

if err != nil {
log.Error(fmt.Sprintf("Error parsing LastUpdateTime %s for template %s: %s", instance.Vm.LastUpdateTime, wantedInstanceTemplate, err.Error()))
}
} else {
instanceTime, err = time.Parse(time.RFC3339, instance.Vm.CreationTime)
if err != nil {
log.Error(fmt.Sprintf("Error parsing CreationTime %s for template %s: %s", instance.Vm.CreationTime, wantedInstanceTemplate, err.Error()))
}
}
thisAge := now.Sub(instanceTime).Seconds()
age = max(age, thisAge)
}
}
}
if _, ok := InstanceAgePerTemplateMaximumMap[wantedState][wantedInstanceTemplate]; !ok {
InstanceAgePerTemplateMaximumMap[wantedState][wantedInstanceTemplate] = int(age)
}
}
}
checkAndHandleResetOfGaugeVecMetric((len(instances) + len(instanceTemplates)), "anka_instance_max_age_per_template_seconds", metric)
for wantedState, wantedStateMap := range InstanceAgePerTemplateMaximumMap {
for wantedTemplateUUID, age := range wantedStateMap {
metric.With(prometheus.Labels{"state": wantedState, "template_uuid": wantedTemplateUUID, "template_name": instanceTemplatesMap[wantedTemplateUUID]}).Set(float64(age))
}
}
},
},
}

func init() { // runs on exporter init only (updates are made with the above EventHandler; triggered by the Client)
Expand Down
14 changes: 8 additions & 6 deletions src/types/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,12 +79,14 @@ type Instance struct {
}

type VmData struct {
State string `json:"instance_state"`
TemplateUUID string `json:"vmid"`
TemplateName string
GroupUUID string `json:"group_id"`
NodeUUID string `json:"node_id"`
Arch string `json:"arch"`
State string `json:"instance_state"`
TemplateUUID string `json:"vmid"`
TemplateName string
GroupUUID string `json:"group_id"`
NodeUUID string `json:"node_id"`
Arch string `json:"arch"`
CreationTime string `json:"cr_time"`
LastUpdateTime string `json:"ts"`
}

type Response interface {
Expand Down
Loading