=6.9.0"
}
@@ -212,7 +210,6 @@
"version": "7.22.20",
"resolved": "https://registry.npmjs.org/@babel/core/-/core-7.22.20.tgz",
"integrity": "sha512-Y6jd1ahLubuYweD/zJH+vvOY141v4f9igNQAQ+MBgq9JlHS2iTsZKn1aMsb3vGccZsXI16VzTBw52Xx0DWmtnA==",
- "dev": true,
"dependencies": {
"@ampproject/remapping": "^2.2.0",
"@babel/code-frame": "^7.22.13",
@@ -241,14 +238,12 @@
"node_modules/@babel/core/node_modules/convert-source-map": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz",
- "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==",
- "dev": true
+ "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A=="
},
"node_modules/@babel/core/node_modules/semver": {
"version": "6.3.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
"integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
- "dev": true,
"bin": {
"semver": "bin/semver.js"
}
@@ -294,7 +289,6 @@
"version": "7.22.15",
"resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.15.tgz",
"integrity": "sha512-y6EEzULok0Qvz8yyLkCvVX+02ic+By2UdOhylwUOvOn9dvYc9mKICJuuU1n1XBI02YWsNsnrY1kc6DVbjcXbtw==",
- "dev": true,
"dependencies": {
"@babel/compat-data": "^7.22.9",
"@babel/helper-validator-option": "^7.22.15",
@@ -310,7 +304,6 @@
"version": "6.3.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
"integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
- "dev": true,
"bin": {
"semver": "bin/semver.js"
}
@@ -447,7 +440,6 @@
"version": "7.22.20",
"resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.22.20.tgz",
"integrity": "sha512-dLT7JVWIUUxKOs1UnJUBR3S70YK+pKX6AbJgB2vMIvEkZkrfJDbYDJesnPshtKV4LhDOR3Oc5YULeDizRek+5A==",
- "dev": true,
"dependencies": {
"@babel/helper-environment-visitor": "^7.22.20",
"@babel/helper-module-imports": "^7.22.15",
@@ -520,7 +512,6 @@
"version": "7.22.5",
"resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.22.5.tgz",
"integrity": "sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==",
- "dev": true,
"dependencies": {
"@babel/types": "^7.22.5"
},
@@ -571,7 +562,6 @@
"version": "7.22.15",
"resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.22.15.tgz",
"integrity": "sha512-bMn7RmyFjY/mdECUbgn9eoSY4vqvacUnS9i9vGAGttgFWesO6B4CYWA7XlpbWgBt71iv/hfbPlynohStqnu5hA==",
- "dev": true,
"engines": {
"node": ">=6.9.0"
}
@@ -594,7 +584,6 @@
"version": "7.22.15",
"resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.22.15.tgz",
"integrity": "sha512-7pAjK0aSdxOwR+CcYAqgWOGy5dcfvzsTIfFTb2odQqW47MDfv14UaJDY6eng8ylM2EaeKXdxaSWESbkmaQHTmw==",
- "dev": true,
"dependencies": {
"@babel/template": "^7.22.15",
"@babel/traverse": "^7.22.15",
@@ -9979,7 +9968,7 @@
"version": "3.19.11",
"resolved": "https://registry.npmjs.org/@types/cytoscape/-/cytoscape-3.19.11.tgz",
"integrity": "sha512-ny4i4BOoZxdc9DrSa9RrasXHPRFgt0PeINgj/CegzKu7CJO+UQP0KnjebYJ+KoLymyUbCX86vmqz5B3LK10w5Q==",
- "dev": true
+ "devOptional": true
},
"node_modules/@types/cytoscape-popper": {
"version": "2.0.3",
@@ -12057,7 +12046,6 @@
"version": "4.21.10",
"resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.10.tgz",
"integrity": "sha512-bipEBdZfVH5/pwrvqc+Ub0kUPVfGUhlKxbvfD+z1BDnPEO/X98ruXGA1WP5ASpAFKan7Qr6j736IacbZQuAlKQ==",
- "dev": true,
"funding": [
{
"type": "opencollective",
@@ -14215,8 +14203,7 @@
"node_modules/electron-to-chromium": {
"version": "1.4.525",
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.525.tgz",
- "integrity": "sha512-GIZ620hDK4YmIqAWkscG4W6RwY6gOx1y5J6f4JUQwctiJrqH2oxZYU4mXHi35oV32tr630UcepBzSBGJ/WYcZA==",
- "dev": true
+ "integrity": "sha512-GIZ620hDK4YmIqAWkscG4W6RwY6gOx1y5J6f4JUQwctiJrqH2oxZYU4mXHi35oV32tr630UcepBzSBGJ/WYcZA=="
},
"node_modules/elliptic": {
"version": "6.5.4",
@@ -14549,7 +14536,6 @@
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
"integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
- "dev": true,
"engines": {
"node": ">=6"
}
@@ -16059,7 +16045,6 @@
"version": "1.0.0-beta.2",
"resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
"integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==",
- "dev": true,
"engines": {
"node": ">=6.9.0"
}
@@ -18954,7 +18939,6 @@
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
"integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
- "dev": true,
"bin": {
"json5": "lib/cli.js"
},
@@ -19265,7 +19249,6 @@
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
"integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
- "dev": true,
"dependencies": {
"yallist": "^3.0.2"
}
@@ -19908,8 +19891,7 @@
"node_modules/node-releases": {
"version": "2.0.13",
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.13.tgz",
- "integrity": "sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==",
- "dev": true
+ "integrity": "sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ=="
},
"node_modules/normalize-package-data": {
"version": "3.0.3",
@@ -24357,7 +24339,6 @@
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz",
"integrity": "sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA==",
- "dev": true,
"funding": [
{
"type": "opencollective",
@@ -25105,8 +25086,7 @@
"node_modules/yallist": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
- "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==",
- "dev": true
+ "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
},
"node_modules/yaml": {
"version": "2.3.2",
diff --git a/dashboard/pages/onboarding/cloud-accounts/index.tsx b/dashboard/pages/onboarding/cloud-accounts/index.tsx
index 476eb9d20..667bcdd1c 100644
--- a/dashboard/pages/onboarding/cloud-accounts/index.tsx
+++ b/dashboard/pages/onboarding/cloud-accounts/index.tsx
@@ -99,7 +99,7 @@ export default function CloudAccounts() {
))}
-
diff --git a/dashboard/pages/onboarding/database/postgres.tsx b/dashboard/pages/onboarding/database/postgres.tsx
index 7c9850148..ff5a6903a 100644
--- a/dashboard/pages/onboarding/database/postgres.tsx
+++ b/dashboard/pages/onboarding/database/postgres.tsx
@@ -51,7 +51,7 @@ export default function PostgreSQLCredentials() {
message:
'Your Postgres database has been successfully connected to Komiser.'
});
- router.push('/onboarding/complete/');
+ router.push('/onboarding/choose-cloud/');
}
});
};
diff --git a/dashboard/pages/onboarding/database/sqlite.tsx b/dashboard/pages/onboarding/database/sqlite.tsx
index de368007e..e21a1afb1 100644
--- a/dashboard/pages/onboarding/database/sqlite.tsx
+++ b/dashboard/pages/onboarding/database/sqlite.tsx
@@ -50,7 +50,7 @@ export default function SqliteCredentials() {
message:
'Your Postgres database has been successfully connected to Komiser.'
});
- router.push('/onboarding/complete/');
+ router.push('/onboarding/choose-cloud/');
}
});
};
diff --git a/dashboard/styles/globals.css b/dashboard/styles/globals.css
index 1578917f7..78028189f 100644
--- a/dashboard/styles/globals.css
+++ b/dashboard/styles/globals.css
@@ -32,7 +32,7 @@
}
}
-@variants responsive {
+@layer responsive {
/* Chrome, Safari and Opera */
.no-scrollbar::-webkit-scrollbar {
display: none;
diff --git a/docker/docker-compose.postgres.yml b/docker/docker-compose.postgres.yml
index d390e39f7..a24ee83df 100644
--- a/docker/docker-compose.postgres.yml
+++ b/docker/docker-compose.postgres.yml
@@ -11,7 +11,7 @@ services:
- 3000:3000
postgres:
- image: postgres:13
+ image: postgres:16
restart: always
environment:
- POSTGRES_USER=root
diff --git a/handlers/accounts_handler.go b/handlers/accounts_handler.go
index 9852c07a6..df5381927 100644
--- a/handlers/accounts_handler.go
+++ b/handlers/accounts_handler.go
@@ -10,7 +10,6 @@ import (
"github.com/BurntSushi/toml"
"github.com/gin-gonic/gin"
- "github.com/sirupsen/logrus"
log "github.com/sirupsen/logrus"
"github.com/tailwarden/komiser/models"
"github.com/tailwarden/komiser/utils"
@@ -33,7 +32,7 @@ func (handler *ApiHandler) IsOnboardedHandler(c *gin.Context) {
}
if handler.db == nil {
- output.Status = "PENDING_ACCOUNTS"
+ output.Status = "PENDING_DATABASE"
c.JSON(http.StatusOK, output)
return
}
@@ -41,7 +40,7 @@ func (handler *ApiHandler) IsOnboardedHandler(c *gin.Context) {
accounts := make([]models.Account, 0)
err := handler.db.NewRaw("SELECT * FROM accounts").Scan(handler.ctx, &accounts)
if err != nil {
- logrus.WithError(err).Error("scan failed")
+ log.WithError(err).Error("scan failed")
c.JSON(http.StatusInternalServerError, gin.H{"error": "scan failed"})
return
}
@@ -65,7 +64,7 @@ func (handler *ApiHandler) ListCloudAccountsHandler(c *gin.Context) {
err := handler.db.NewRaw("SELECT * FROM accounts").Scan(handler.ctx, &accounts)
if err != nil {
- logrus.WithError(err).Error("scan failed")
+ log.WithError(err).Error("scan failed")
c.JSON(http.StatusInternalServerError, gin.H{"error": "scan failed"})
return
}
@@ -113,6 +112,7 @@ func (handler *ApiHandler) NewCloudAccountHandler(c *gin.Context) {
accountId, _ := result.LastInsertId()
account.Id = accountId
+ go fetchResourcesForAccount(c, account, handler.db, []string{})
}
if handler.telemetry {
diff --git a/handlers/helper.go b/handlers/helper.go
new file mode 100644
index 000000000..615f3e8b4
--- /dev/null
+++ b/handlers/helper.go
@@ -0,0 +1,345 @@
+package handlers
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "os"
+ "sync"
+ "time"
+
+ tccommon "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
+ tccvm "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm/v20170312"
+
+ "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+ awsConfig "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/civo/civogo"
+ "github.com/digitalocean/godo"
+ "github.com/linode/linodego"
+ "github.com/mongodb-forks/digest"
+ "github.com/oracle/oci-go-sdk/common"
+ ovhPkg "github.com/ovh/go-ovh/ovh"
+ "github.com/scaleway/scaleway-sdk-go/scw"
+ log "github.com/sirupsen/logrus"
+ "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
+ "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/regions"
+ "github.com/uptrace/bun"
+ mdb "go.mongodb.org/atlas/mongodbatlas"
+ "golang.org/x/oauth2"
+ "golang.org/x/oauth2/google"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/tools/clientcmd"
+
+ "github.com/getsentry/sentry-go"
+ "github.com/tailwarden/komiser/models"
+ "github.com/tailwarden/komiser/providers"
+ "github.com/tailwarden/komiser/providers/aws"
+ "github.com/tailwarden/komiser/providers/azure"
+ "github.com/tailwarden/komiser/providers/civo"
+ do "github.com/tailwarden/komiser/providers/digitalocean"
+ "github.com/tailwarden/komiser/providers/gcp"
+ "github.com/tailwarden/komiser/providers/k8s"
+ "github.com/tailwarden/komiser/providers/linode"
+ "github.com/tailwarden/komiser/providers/mongodbatlas"
+ "github.com/tailwarden/komiser/providers/oci"
+ "github.com/tailwarden/komiser/providers/ovh"
+ "github.com/tailwarden/komiser/providers/scaleway"
+ "github.com/tailwarden/komiser/providers/tencent"
+ "github.com/tailwarden/komiser/utils"
+)
+
+func triggerFetchingWorkflow(ctx context.Context, client providers.ProviderClient, provider string, db *bun.DB, regions []string, wp *providers.WorkerPool) {
+ localHub := sentry.CurrentHub().Clone()
+
+ defer func() {
+ err := recover()
+ if err != nil {
+ log.WithField("err", err).Error(fmt.Sprintf("error fetching %s resources", provider))
+ localHub.CaptureException(err.(error))
+ localHub.Flush(2 * time.Second)
+ }
+ }()
+
+ localHub.ConfigureScope(func(scope *sentry.Scope) {
+ scope.SetTag("provider", provider)
+ })
+
+ var analytics utils.Analytics
+ telemetry := false
+ switch provider {
+ case "AWS":
+ aws.FetchResources(ctx, client, regions, db, telemetry, analytics, wp)
+ case "DigitalOcean":
+ do.FetchResources(ctx, client, db, telemetry, analytics, wp)
+ case "OCI":
+ oci.FetchResources(ctx, client, db, telemetry, analytics, wp)
+ case "Civo":
+ civo.FetchResources(ctx, client, db, telemetry, analytics, wp)
+ case "Kubernetes":
+ k8s.FetchResources(ctx, client, db, telemetry, analytics, wp)
+ case "Linode":
+ linode.FetchResources(ctx, client, db, telemetry, analytics, wp)
+ case "Tencent":
+ tencent.FetchResources(ctx, client, db, telemetry, analytics, wp)
+ case "Azure":
+ azure.FetchResources(ctx, client, db, telemetry, analytics, wp)
+ case "Scaleway":
+ scaleway.FetchResources(ctx, client, db, telemetry, analytics, wp)
+ case "MongoDBAtlas":
+ mongodbatlas.FetchResources(ctx, client, db, telemetry, analytics, wp)
+ case "GCP":
+ gcp.FetchResources(ctx, client, db, telemetry, analytics, wp)
+ case "OVH":
+ ovh.FetchResources(ctx, client, db, telemetry, analytics, wp)
+ }
+}
+
+func fetchResourcesForAccount(ctx context.Context, account models.Account, db *bun.DB, regions []string) {
+ numWorkers := 64
+ wp := providers.NewWorkerPool(numWorkers)
+ wp.Start()
+
+ var wwg sync.WaitGroup
+ workflowTrigger := func(client providers.ProviderClient, provider string) {
+ wwg.Add(1)
+ go func() {
+ defer wwg.Done()
+ triggerFetchingWorkflow(ctx, client, provider, db, regions, wp)
+ }()
+ }
+
+ client, err := makeClientFromAccount(account)
+ if err != nil {
+ log.Error(err)
+ return
+ }
+ if client.AWSClient != nil {
+ workflowTrigger(*client, "AWS")
+ } else if client.DigitalOceanClient != nil {
+ workflowTrigger(*client, "DigitalOcean")
+ } else if client.OciClient != nil {
+ workflowTrigger(*client, "OCI")
+ } else if client.CivoClient != nil {
+ workflowTrigger(*client, "Civo")
+ } else if client.K8sClient != nil {
+ workflowTrigger(*client, "Kubernetes")
+ } else if client.LinodeClient != nil {
+ workflowTrigger(*client, "Linode")
+ } else if client.TencentClient != nil {
+ workflowTrigger(*client, "Tencent")
+ } else if client.AzureClient != nil {
+ workflowTrigger(*client, "Azure")
+ } else if client.ScalewayClient != nil {
+ workflowTrigger(*client, "Scaleway")
+ } else if client.MongoDBAtlasClient != nil {
+ workflowTrigger(*client, "MongoDBAtlas")
+ } else if client.GCPClient != nil {
+ workflowTrigger(*client, "GCP")
+ } else if client.OVHClient != nil {
+ workflowTrigger(*client, "OVH")
+ }
+
+ wwg.Wait()
+ wp.Wait()
+}
+
+func makeClientFromAccount(account models.Account) (*providers.ProviderClient, error) {
+ if account.Provider == "aws" {
+ if account.Credentials["source"] == "credentials-file" {
+ if len(account.Credentials["path"]) > 0 {
+ cfg, err := awsConfig.LoadDefaultConfig(context.Background(), awsConfig.WithSharedConfigProfile(account.Credentials["profile"]), awsConfig.WithSharedCredentialsFiles(
+ []string{account.Credentials["path"]},
+ ))
+ if err != nil {
+ return nil, err
+ }
+ return &providers.ProviderClient{
+ AWSClient: &cfg,
+ Name: account.Name,
+ }, nil
+ } else {
+ cfg, err := awsConfig.LoadDefaultConfig(context.Background(), awsConfig.WithSharedConfigProfile(account.Credentials["profile"]))
+ if err != nil {
+ return nil, err
+ }
+ return &providers.ProviderClient{
+ AWSClient: &cfg,
+ Name: account.Name,
+ }, err
+ }
+ } else if account.Credentials["source"] == "environment-variables" {
+ cfg, err := awsConfig.LoadDefaultConfig(context.Background())
+ if err != nil {
+ return nil, err
+ }
+ return &providers.ProviderClient{
+ AWSClient: &cfg,
+ Name: account.Name,
+ }, nil
+ }
+ }
+
+ if account.Provider == "digitalocean" {
+ client := godo.NewFromToken(account.Credentials["token"])
+ return &providers.ProviderClient{
+ DigitalOceanClient: client,
+ Name: account.Name,
+ }, nil
+ }
+
+ if account.Provider == "oci" {
+ if account.Credentials["source"] == "CREDENTIALS_FILE" {
+ return &providers.ProviderClient{
+ OciClient: common.DefaultConfigProvider(),
+ Name: account.Name,
+ }, nil
+ }
+ }
+
+ if account.Provider == "civo" {
+ client, err := civogo.NewClient(account.Credentials["token"], "LON1")
+ if err != nil {
+ return nil, err
+ }
+ return &providers.ProviderClient{
+ CivoClient: client,
+ Name: account.Name,
+ }, nil
+ }
+
+ if account.Provider == "kubernetes" {
+ kubeConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+ &clientcmd.ClientConfigLoadingRules{ExplicitPath: account.Credentials["path"]},
+ &clientcmd.ConfigOverrides{}).ClientConfig()
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ k8sClient, err := kubernetes.NewForConfig(kubeConfig)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ client := providers.K8sClient{
+ Client: k8sClient,
+ OpencostBaseUrl: account.Credentials["opencostbaseurl"],
+ }
+
+ return &providers.ProviderClient{
+ K8sClient: &client,
+ Name: account.Name,
+ }, nil
+ }
+
+ if account.Provider == "linode" {
+ tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: account.Credentials["token"]})
+ oauth2Client := &http.Client{
+ Transport: &oauth2.Transport{
+ Source: tokenSource,
+ },
+ }
+ client := linodego.NewClient(oauth2Client)
+ return &providers.ProviderClient{
+ LinodeClient: &client,
+ Name: account.Name,
+ }, nil
+ }
+
+ if account.Provider == "tencent" {
+ credential := tccommon.NewCredential(account.Credentials["secretId"], account.Credentials["secretKey"])
+ cpf := profile.NewClientProfile()
+ cpf.Language = "en-US"
+ client, err := tccvm.NewClient(credential, regions.Frankfurt, cpf)
+ if err != nil {
+ return nil, err
+ }
+
+ return &providers.ProviderClient{
+ TencentClient: client,
+ Name: account.Name,
+ }, nil
+ }
+
+ if account.Provider == "azure" {
+ creds, err := azidentity.NewClientSecretCredential(account.Credentials["tenantId"], account.Credentials["clientId"], account.Credentials["clientSecret"], &azidentity.ClientSecretCredentialOptions{})
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ client := providers.AzureClient{
+ Credentials: creds,
+ SubscriptionId: account.Credentials["subscriptionId"],
+ }
+
+ return &providers.ProviderClient{
+ AzureClient: &client,
+ Name: account.Name,
+ }, nil
+ }
+
+ if account.Provider == "scaleway" {
+ client, err := scw.NewClient(
+ scw.WithDefaultOrganizationID(account.Credentials["organizationId"]),
+ scw.WithAuth(account.Credentials["accessKey"], account.Credentials["secretKey"]),
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ return &providers.ProviderClient{
+ ScalewayClient: client,
+ Name: account.Name,
+ }, nil
+ }
+
+ if account.Provider == "mongodb" {
+ t := digest.NewTransport(account.Credentials["publicApiKey"], account.Credentials["privateApiKey"])
+ tc, err := t.Client()
+ if err != nil {
+ log.Fatal(err.Error())
+ }
+
+ client := mdb.NewClient(tc)
+ return &providers.ProviderClient{
+ MongoDBAtlasClient: client,
+ Name: account.Name,
+ }, nil
+ }
+
+ if account.Provider == "gcp" {
+ data, err := os.ReadFile(account.Credentials["accountKey"])
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ creds, err := google.CredentialsFromJSON(context.Background(), data, "https://www.googleapis.com/auth/cloud-platform")
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ return &providers.ProviderClient{
+ GCPClient: &providers.GCPClient{
+ Credentials: creds,
+ },
+ Name: account.Name,
+ }, nil
+ }
+
+ if account.Provider == "ovh" {
+ client, err := ovhPkg.NewClient(
+ account.Credentials["endpoint"],
+ account.Credentials["applicationKey"],
+ account.Credentials["applicationSecret"],
+ account.Credentials["consumerKey"],
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ return &providers.ProviderClient{
+ OVHClient: client,
+ Name: account.Name,
+ }, nil
+ }
+ return nil, fmt.Errorf("provider not supported")
+}
diff --git a/internal/internal.go b/internal/internal.go
index 388488bf3..2b8235e24 100644
--- a/internal/internal.go
+++ b/internal/internal.go
@@ -210,7 +210,7 @@ func setupDBConnection(c *models.Config) error {
return nil
}
-func triggerFetchingWorfklow(ctx context.Context, client providers.ProviderClient, provider string, telemetry bool, regions []string, wp *providers.WorkerPool) {
+func triggerFetchingWorkflow(ctx context.Context, client providers.ProviderClient, provider string, telemetry bool, regions []string, wp *providers.WorkerPool) {
localHub := sentry.CurrentHub().Clone()
defer func() {
@@ -270,7 +270,7 @@ func fetchResources(ctx context.Context, clients []providers.ProviderClient, reg
wwg.Add(1)
go func() {
defer wwg.Done()
- triggerFetchingWorfklow(ctx, client, provider, telemetry, regions, wp)
+ triggerFetchingWorkflow(ctx, client, provider, telemetry, regions, wp)
}()
}
diff --git a/providers/aws/aws.go b/providers/aws/aws.go
index 5217c3bf5..a5d740ad2 100644
--- a/providers/aws/aws.go
+++ b/providers/aws/aws.go
@@ -2,12 +2,11 @@ package aws
import (
"context"
+ "encoding/json"
"strings"
"time"
- "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/costexplorer"
- "github.com/aws/aws-sdk-go-v2/service/costexplorer/types"
log "github.com/sirupsen/logrus"
"github.com/tailwarden/komiser/models"
"github.com/tailwarden/komiser/providers"
@@ -118,41 +117,27 @@ func FetchResources(ctx context.Context, client providers.ProviderClient, region
listOfSupportedRegions = regions
}
- costexplorerClient := costexplorer.NewFromConfig(*client.AWSClient)
- costexplorerOutputList := []*costexplorer.GetCostAndUsageOutput{}
- var nextPageToken *string
- for {
- costexplorerOutput, err := costexplorerClient.GetCostAndUsage(ctx, &costexplorer.GetCostAndUsageInput{
- Granularity: "DAILY",
- Metrics: []string{"UnblendedCost"},
- TimePeriod: &types.DateInterval{
- Start: aws.String(utils.BeginningOfMonth(time.Now()).Format("2006-01-02")),
- End: aws.String(time.Now().Format("2006-01-02")),
- },
- GroupBy: []types.GroupDefinition{
- {
- Key: aws.String("SERVICE"),
- Type: "DIMENSION",
- },
- {
- Key: aws.String("REGION"),
- Type: "DIMENSION",
- },
- },
- NextPageToken: nextPageToken,
- })
+ var costexplorerOutputList []*costexplorer.GetCostAndUsageOutput
+ if jsonData, err := readCostExplorerCache(); err == nil {
+ err := json.Unmarshal(jsonData, &costexplorerOutputList)
if err != nil {
- log.Warn("Couldn't fetch cost and usage data:", err)
- break
+ log.Warn("Failed to unmarshal cached cost explorer data:", err)
}
-
- costexplorerOutputList = append(costexplorerOutputList, costexplorerOutput)
-
- if aws.ToString(costexplorerOutput.NextPageToken) == "" {
- break
+ } else {
+ costexplorerOutputList, err = getCostexplorerOutput(
+ ctx, client, utils.BeginningMonthsAgo(time.Now(), 6).Format("2006-01-02"), utils.EndingOfLastMonth(time.Now()).Format("2006-01-02"),
+ )
+ if err != nil {
+ log.Warn("Failed to get cost explorer output:", err)
}
+ if err := writeCostExplorerCache(costexplorerOutputList); err != nil {
+ log.Warn("Failed to write cost explorer cache:", err)
+ }
+ }
- nextPageToken = costexplorerOutput.NextPageToken
+ costexplorerOutputList, err := getCostexplorerOutput(ctx, client, utils.BeginningOfMonth(time.Now()).Format("2006-01-02"), time.Now().Format("2006-01-02"))
+ if err != nil {
+ log.Warn("Failed to get cost explorer output:", err)
}
ctxWithCostexplorerOutput := context.WithValue(ctx, awsUtils.CostexplorerKey, costexplorerOutputList)
for _, region := range listOfSupportedRegions {
diff --git a/providers/aws/costexplorer.go b/providers/aws/costexplorer.go
new file mode 100644
index 000000000..eee2f18e9
--- /dev/null
+++ b/providers/aws/costexplorer.go
@@ -0,0 +1,77 @@
+package aws
+
+import (
+ "context"
+ "encoding/json"
+ "os"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/service/costexplorer"
+ "github.com/aws/aws-sdk-go-v2/service/costexplorer/types"
+ log "github.com/sirupsen/logrus"
+ "github.com/tailwarden/komiser/providers"
+)
+
+const costExplorerCacheFile = "cost_explorer_cache.json"
+
+// readCostExplorerCache reads cost explorer cache data from the file.
+func readCostExplorerCache() ([]byte, error) {
+ file, err := os.ReadFile(costExplorerCacheFile)
+ if err != nil {
+ return nil, err
+ }
+ return file, nil
+}
+
+// writeCostExplorerCache writes cost explorer cache data to the file.
+func writeCostExplorerCache(data interface{}) error {
+ jsonData, err := json.Marshal(data)
+ if err != nil {
+ return err
+ }
+ err = os.WriteFile(costExplorerCacheFile, jsonData, 0644)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func getCostexplorerOutput(ctx context.Context, client providers.ProviderClient, start, end string) ([]*costexplorer.GetCostAndUsageOutput, error) {
+ costexplorerOutputList := []*costexplorer.GetCostAndUsageOutput{}
+ costexplorerClient := costexplorer.NewFromConfig(*client.AWSClient)
+ var nextPageToken *string
+ for {
+ costexplorerOutput, err := costexplorerClient.GetCostAndUsage(ctx, &costexplorer.GetCostAndUsageInput{
+ Granularity: "DAILY",
+ Metrics: []string{"UnblendedCost"},
+ TimePeriod: &types.DateInterval{
+ Start: aws.String(start),
+ End: aws.String(end),
+ },
+ GroupBy: []types.GroupDefinition{
+ {
+ Key: aws.String("SERVICE"),
+ Type: "DIMENSION",
+ },
+ {
+ Key: aws.String("REGION"),
+ Type: "DIMENSION",
+ },
+ },
+ NextPageToken: nextPageToken,
+ })
+ if err != nil {
+ log.Warn("Couldn't fetch cost and usage data:", err)
+ return nil, err
+ }
+
+ costexplorerOutputList = append(costexplorerOutputList, costexplorerOutput)
+
+ if aws.ToString(costexplorerOutput.NextPageToken) == "" {
+ break
+ }
+
+ nextPageToken = costexplorerOutput.NextPageToken
+ }
+ return costexplorerOutputList, nil
+}
diff --git a/providers/aws/rds/auto_backups.go b/providers/aws/rds/auto_backups.go
index 73c256622..777bec933 100644
--- a/providers/aws/rds/auto_backups.go
+++ b/providers/aws/rds/auto_backups.go
@@ -3,9 +3,9 @@ package rds
import (
"context"
"fmt"
- "github.com/aws/aws-sdk-go-v2/aws"
"time"
+ "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/rds"
log "github.com/sirupsen/logrus"
"github.com/tailwarden/komiser/models"
@@ -36,6 +36,10 @@ func AutoBackups(ctx context.Context, client providers.ProviderClient) ([]models
Name: _backupName,
FetchedAt: time.Now(),
Link: fmt.Sprintf("https:/%s.console.aws.amazon.com/rds/home?region=%s#dbinstance:id=%s", client.AWSClient.Region, client.AWSClient.Region, *backup.DBInstanceIdentifier),
+ Metadata: map[string]string{
+ "Engine": *backup.Engine,
+ "EngineVersion": *backup.EngineVersion,
+ },
})
}
diff --git a/providers/aws/rds/cluster_snapshot.go b/providers/aws/rds/cluster_snapshot.go
index 57300aa18..0766b4021 100644
--- a/providers/aws/rds/cluster_snapshot.go
+++ b/providers/aws/rds/cluster_snapshot.go
@@ -57,6 +57,10 @@ func ClusterSnapshots(ctx context.Context, client providers.ProviderClient) ([]m
FetchedAt: time.Now(),
Tags: tags,
Link: fmt.Sprintf("https:/%s.console.aws.amazon.com/rds/home?region=%s#snapshots-list:id=%s", client.AWSClient.Region, client.AWSClient.Region, *clusterSnapshot.DBClusterSnapshotIdentifier),
+ Metadata: map[string]string{
+ "Engine": *clusterSnapshot.Engine,
+ "EngineVersion": *clusterSnapshot.EngineVersion,
+ },
})
}
diff --git a/providers/aws/rds/clusters.go b/providers/aws/rds/clusters.go
index 0877e7f49..ec3d398e4 100644
--- a/providers/aws/rds/clusters.go
+++ b/providers/aws/rds/clusters.go
@@ -51,6 +51,10 @@ func Clusters(ctx context.Context, client ProviderClient) ([]Resource, error) {
FetchedAt: time.Now(),
Tags: tags,
Link: fmt.Sprintf("https:/%s.console.aws.amazon.com/rds/home?region=%s#database:id=%s", client.AWSClient.Region, client.AWSClient.Region, *cluster.DBClusterIdentifier),
+ Metadata: map[string]string{
+ "Engine": *cluster.Engine,
+ "EngineVersion": *cluster.EngineVersion,
+ },
})
}
diff --git a/providers/aws/rds/instances.go b/providers/aws/rds/instances.go
index 9a30ee538..6515ed4f6 100644
--- a/providers/aws/rds/instances.go
+++ b/providers/aws/rds/instances.go
@@ -122,12 +122,14 @@ func Instances(ctx context.Context, client providers.ProviderClient) ([]models.R
ResourceId: *instance.DBInstanceArn,
Cost: monthlyCost,
Metadata: map[string]string{
- "serviceCost": fmt.Sprint(serviceCost),
+ "serviceCost": fmt.Sprint(serviceCost),
+ "engine": *instance.Engine,
+ "engineVersion": *instance.EngineVersion,
},
- Name: _instanceName,
- FetchedAt: time.Now(),
- Tags: tags,
- Link: fmt.Sprintf("https:/%s.console.aws.amazon.com/rds/home?region=%s#database:id=%s", client.AWSClient.Region, client.AWSClient.Region, *instance.DBInstanceIdentifier),
+ Name: _instanceName,
+ FetchedAt: time.Now(),
+ Tags: tags,
+ Link: fmt.Sprintf("https:/%s.console.aws.amazon.com/rds/home?region=%s#database:id=%s", client.AWSClient.Region, client.AWSClient.Region, *instance.DBInstanceIdentifier),
})
}
diff --git a/providers/aws/utils/utils.go b/providers/aws/utils/utils.go
index bcec79e9a..fe5450e76 100644
--- a/providers/aws/utils/utils.go
+++ b/providers/aws/utils/utils.go
@@ -25,7 +25,7 @@ func GetCostAndUsage(ctx context.Context, region string, svcName string) (float6
for _, costexplorerOutput := range costexplorerOutputList {
for _, group := range costexplorerOutput.ResultsByTime {
for _, v := range group.Groups {
- if v.Keys[0] == svcName && v.Keys[1] == region {
+ if v.Keys[0] == svcName {
amt, err := strconv.ParseFloat(*v.Metrics["UnblendedCost"].Amount, 64)
if err != nil {
return 0, err
diff --git a/utils/datecalc.go b/utils/datecalc.go
index 242add07a..2ba7cb2e4 100644
--- a/utils/datecalc.go
+++ b/utils/datecalc.go
@@ -17,3 +17,19 @@ func EndingOfMonth(date time.Time) time.Time {
}
return endingOfMonth
}
+
+func BeginningMonthsAgo(date time.Time, months int) time.Time {
+ beginningOfTargetMonth, err := time.Parse("2006-01-02", date.AddDate(0, -months, -date.Day()+1).Format("2006-01-02"))
+ if err != nil {
+ return time.Now()
+ }
+ return beginningOfTargetMonth
+}
+
+func EndingOfLastMonth(date time.Time) time.Time {
+ endingOfMonth, err := time.Parse("2006-01-02", date.AddDate(0, 0, -date.Day()+1).Format("2006-01-02"))
+ if err != nil {
+ return time.Now()
+ }
+ return endingOfMonth
+}