From a628b01edaa036a773df43b3d3fcbec6537bc6f4 Mon Sep 17 00:00:00 2001 From: Heba Elayoty <31887807+helayoty@users.noreply.github.com> Date: Mon, 23 Jan 2023 19:06:56 -0800 Subject: [PATCH] refactor: Migrate ACI to Azure SDK v2 - Part 1 (#422) --- cmd/virtual-kubelet/main.go | 5 +- go.mod | 27 +- go.sum | 52 ++-- pkg/analytics/analytics.go | 20 +- pkg/analytics/analytics_test.go | 6 +- pkg/auth/auth.go | 35 ++- pkg/client/client.go | 56 ++-- pkg/client/client_apis.go | 293 ++++++++++--------- pkg/client/extensions.go | 70 ++--- pkg/client/interfaces.go | 3 +- pkg/metrics/metrics.go | 27 +- pkg/metrics/metrics_test.go | 23 +- pkg/metrics/mock_metrics_test.go | 5 +- pkg/network/aci_network.go | 31 +- pkg/network/aci_network_test.go | 2 +- pkg/provider/aci.go | 312 +++++++++++---------- pkg/provider/aci_init_container_test.go | 12 +- pkg/provider/aci_test.go | 259 +++++++++++------ pkg/provider/aci_volumes.go | 32 +-- pkg/provider/aci_volumes_test.go | 50 ++-- pkg/provider/containergroup_to_pod.go | 96 ++++--- pkg/provider/containergroup_to_pod_test.go | 7 +- pkg/provider/mock_aci_test.go | 24 +- pkg/tests/utils.go | 152 +++++++--- pkg/util/aci_utils.go | 21 +- pkg/validation/validator.go | 60 ++-- 26 files changed, 985 insertions(+), 695 deletions(-) diff --git a/cmd/virtual-kubelet/main.go b/cmd/virtual-kubelet/main.go index e8db6c41..3afdf287 100644 --- a/cmd/virtual-kubelet/main.go +++ b/cmd/virtual-kubelet/main.go @@ -82,7 +82,10 @@ func main() { log.G(ctx).Fatal(err) } - azACIAPIs = client.NewAzClientsAPIs(ctx, azConfig) + azACIAPIs, err = client.NewAzClientsAPIs(ctx, azConfig) + if err != nil { + log.G(ctx).Fatal(err) + } } run := func(ctx context.Context) error { node, err := cli.New(ctx, diff --git a/go.mod b/go.mod index 026cafb0..2227e686 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,11 @@ require ( contrib.go.opencensus.io/exporter/ocagent v0.4.12 github.com/Azure/azure-sdk-for-go v63.0.0+incompatible github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2 v2.2.0-beta.1 github.com/Azure/go-autorest/autorest v0.11.24 github.com/Azure/go-autorest/autorest/adal v0.9.20 github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 - github.com/Azure/go-autorest/autorest/date v0.3.0 github.com/Azure/go-autorest/autorest/mocks v0.4.2 github.com/BurntSushi/toml v0.3.1 github.com/dimchansky/utfbom v1.1.1 @@ -34,12 +35,15 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -54,7 +58,7 @@ require ( github.com/go-openapi/spec v0.19.3 // indirect github.com/go-openapi/swag v0.19.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.2.0 // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.5 // indirect @@ -64,24 +68,26 @@ require ( github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/imdario/mergo v0.3.10 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/json-iterator/go v1.1.11 // indirect + github.com/json-iterator/go v1.1.10 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.11.1 // indirect + github.com/prometheus/client_golang v1.7.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.26.0 // indirect - github.com/prometheus/procfs v0.6.0 // indirect + github.com/prometheus/common v0.10.0 // indirect + github.com/prometheus/procfs v0.1.3 // indirect github.com/spf13/cobra v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88 // indirect - golang.org/x/net v0.0.0-20220906165146-f3363e06e74c // indirect + golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 // indirect - golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect + golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect @@ -92,7 +98,7 @@ require ( google.golang.org/protobuf v1.26.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect k8s.io/apiserver v0.19.10 // indirect k8s.io/component-base v0.19.10 // indirect k8s.io/klog v1.0.0 // indirect @@ -103,6 +109,3 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.0.3 // indirect sigs.k8s.io/yaml v1.2.0 // indirect ) - -// fixinate the azure SDK version for consistency accross all project packages -replace github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v63.0.0+incompatible diff --git a/go.sum b/go.sum index 33204a28..4e7ad4ab 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,12 @@ github.com/Azure/azure-sdk-for-go v63.0.0+incompatible h1:whPsa+jCHQSo5wGMPNLw4b github.com/Azure/azure-sdk-for-go v63.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2 v2.2.0-beta.1 h1:eY6fhA944YceJrJ9OGn1T5iqe5DA2rQ+O1/Gi3P4bXU= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2 v2.2.0-beta.1/go.mod h1:5Q/hN8CkM0y7bBldgIdoPMp9jyBZ1KVeexQvfY2KXw8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -54,6 +60,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= +github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -73,7 +81,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -117,6 +124,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= @@ -150,10 +158,8 @@ github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.3.0 h1:q4c+kbcR0d5rSurhBR8dIgieOaYpXtsdTYfx22Cu6rs= @@ -215,8 +221,9 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -240,7 +247,6 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -252,7 +258,6 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -300,16 +305,13 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -326,6 +328,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -353,7 +357,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -383,6 +386,8 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -394,9 +399,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= -github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -407,16 +411,14 @@ github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7q github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -556,7 +558,6 @@ golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -564,8 +565,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220906165146-f3363e06e74c h1:yKufUcDwucU5urd+50/Opbt4AYpqthk7wHpHok8f1lo= -golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -578,7 +579,6 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -611,19 +611,16 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -754,8 +751,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= diff --git a/pkg/analytics/analytics.go b/pkg/analytics/analytics.go index 9f2be976..36006786 100644 --- a/pkg/analytics/analytics.go +++ b/pkg/analytics/analytics.go @@ -6,17 +6,23 @@ import ( "fmt" "os" - azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" +) + +const ( + LogAnalyticsMetadataKeyPodUUID string = "pod-uuid" + LogAnalyticsMetadataKeyNodeName string = "node-name" + LogAnalyticsMetadataKeyClusterResourceID string = "cluster-resource-id" ) // NewContainerGroupDiagnostics creates a container group diagnostics object -func NewContainerGroupDiagnostics(logAnalyticsID, logAnalyticsKey string) (*azaci.ContainerGroupDiagnostics, error) { +func NewContainerGroupDiagnostics(logAnalyticsID, logAnalyticsKey string) (*azaciv2.ContainerGroupDiagnostics, error) { if logAnalyticsID == "" || logAnalyticsKey == "" { return nil, errors.New("log Analytics configuration requires both the workspace ID and Key") } - return &azaci.ContainerGroupDiagnostics{ - LogAnalytics: &azaci.LogAnalytics{ + return &azaciv2.ContainerGroupDiagnostics{ + LogAnalytics: &azaciv2.LogAnalytics{ WorkspaceID: &logAnalyticsID, WorkspaceKey: &logAnalyticsKey, }, @@ -24,7 +30,7 @@ func NewContainerGroupDiagnostics(logAnalyticsID, logAnalyticsKey string) (*azac } // NewContainerGroupDiagnosticsFromFile creates a container group diagnostics object from the specified file -func NewContainerGroupDiagnosticsFromFile(filepath string) (*azaci.ContainerGroupDiagnostics, error) { +func NewContainerGroupDiagnosticsFromFile(filepath string) (*azaciv2.ContainerGroupDiagnostics, error) { analyticsDataFile, err := os.Open(filepath) defer analyticsDataFile.Close() if err != nil { @@ -45,12 +51,12 @@ func NewContainerGroupDiagnosticsFromFile(filepath string) (*azaci.ContainerGrou } // Unmarshal the log analytics file. - var logAnalytics azaci.LogAnalytics + var logAnalytics azaciv2.LogAnalytics if err := json.Unmarshal(analyticsData, &logAnalytics); err != nil { return nil, err } - return &azaci.ContainerGroupDiagnostics{ + return &azaciv2.ContainerGroupDiagnostics{ LogAnalytics: &logAnalytics, }, err } diff --git a/pkg/analytics/analytics_test.go b/pkg/analytics/analytics_test.go index 56de3559..d50bdab0 100644 --- a/pkg/analytics/analytics_test.go +++ b/pkg/analytics/analytics_test.go @@ -46,8 +46,10 @@ func TestLogAnalyticsFileParsingSuccess(t *testing.T) { diagnostics, err := NewContainerGroupDiagnosticsFromFile(os.Getenv("LOG_ANALYTICS_AUTH_LOCATION")) assert.Equal(t, err, nil) - assert.Check(t, diagnostics != nil || diagnostics.LogAnalytics != nil, "Unexpected nil diagnostics. Log Analytics file not parsed correctly") - assert.Check(t, *diagnostics.LogAnalytics.WorkspaceID != "" || *diagnostics.LogAnalytics.WorkspaceKey != "", "Unexpected empty analytics authentication credentials. Log Analytics file not parsed correctly") + assert.Check(t, diagnostics != nil || diagnostics.LogAnalytics != nil, + "Unexpected nil diagnostics. Log Analytics file not parsed correctly") + assert.Check(t, diagnostics.LogAnalytics.WorkspaceID != nil || diagnostics.LogAnalytics.WorkspaceKey != nil, + "Unexpected empty analytics authentication credentials. Log Analytics file not parsed correctly") } func TestLogAnalyticsFileParsingFailure(t *testing.T) { diff --git a/pkg/auth/auth.go b/pkg/auth/auth.go index fe38014c..a4e66be7 100644 --- a/pkg/auth/auth.go +++ b/pkg/auth/auth.go @@ -11,8 +11,10 @@ import ( "strings" "unicode/utf16" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" _ "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/adal" "github.com/dimchansky/utfbom" @@ -36,6 +38,38 @@ type Config struct { Authorizer autorest.Authorizer } +// GetMSICredential retrieve MSI credential +func (c *Config) GetMSICredential(ctx context.Context) (*azidentity.ManagedIdentityCredential, error) { + log.G(ctx).Debug("getting token using user identity") + opts := &azidentity.ManagedIdentityCredentialOptions{ + ID: azidentity.ClientID(c.AuthConfig.UserIdentityClientId), + ClientOptions: azcore.ClientOptions{ + Cloud: c.Cloud, + }} + msiCredential, err := azidentity.NewManagedIdentityCredential(opts) + if err != nil { + return nil, err + } + + return msiCredential, nil +} + +// GetSPCredential retrieve SP credential +func (c *Config) GetSPCredential(ctx context.Context) (*azidentity.ClientSecretCredential, error) { + log.G(ctx).Debug("getting token using service principal") + opts := &azidentity.ClientSecretCredentialOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: c.Cloud, + }, + } + spCredential, err := azidentity.NewClientSecretCredential(c.AuthConfig.TenantID, c.AuthConfig.ClientID, c.AuthConfig.ClientSecret, opts) + if err != nil { + return nil, err + } + + return spCredential, nil +} + // getAuthorizer return autorest authorizer. func (c *Config) getAuthorizer(ctx context.Context, resource string) (autorest.Authorizer, error) { var auth autorest.Authorizer @@ -66,7 +100,6 @@ func (c *Config) getAuthorizer(ctx context.Context, resource string) (autorest.A return nil, err } } - auth = autorest.NewBearerAuthorizer(token) return auth, err } diff --git a/pkg/client/client.go b/pkg/client/client.go index 096b99de..2523c46e 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -6,7 +6,9 @@ import ( "fmt" "net/http" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + "github.com/Azure/go-autorest/autorest" "github.com/virtual-kubelet/virtual-kubelet/log" ) @@ -17,14 +19,14 @@ const ( ) type ContainerGroupPropertiesWrapper struct { - ContainerGroupProperties *azaci.ContainerGroupProperties - Extensions []*Extension `json:"extensions,omitempty"` + ContainerGroupProperties *azaciv2.ContainerGroupProperties + Extensions []*azaciv2.DeploymentExtensionSpec } type ContainerGroupWrapper struct { autorest.Response `json:"-"` // Identity - The identity of the container group, if configured. - Identity *azaci.ContainerGroupIdentity `json:"identity,omitempty"` + Identity *azaciv2.ContainerGroupIdentity `json:"identity,omitempty"` // ContainerGroupProperties - The container group properties ContainerGroupPropertiesWrapper *ContainerGroupPropertiesWrapper `json:"properties,omitempty"` // ID - READ-ONLY; The resource id. @@ -98,41 +100,41 @@ func (c *ContainerGroupsClientWrapper) createOrUpdatePreparerWrapper(ctx context // MarshalJSON is the custom marshal for ContainerGroupProperties. func (cg ContainerGroupPropertiesWrapper) MarshalJSON() ([]byte, error) { objectMap := make(map[string]interface{}) - if cg.ContainerGroupProperties.Containers != nil { - objectMap["containers"] = cg.ContainerGroupProperties.Containers + if cg.ContainerGroupProperties.Properties.Containers != nil { + objectMap["containers"] = cg.ContainerGroupProperties.Properties.Containers } - if cg.ContainerGroupProperties.ImageRegistryCredentials != nil { - objectMap["imageRegistryCredentials"] = cg.ContainerGroupProperties.ImageRegistryCredentials + if cg.ContainerGroupProperties.Properties.ImageRegistryCredentials != nil { + objectMap["imageRegistryCredentials"] = cg.ContainerGroupProperties.Properties.ImageRegistryCredentials } - if cg.ContainerGroupProperties.RestartPolicy != "" { - objectMap["restartPolicy"] = cg.ContainerGroupProperties.RestartPolicy + if cg.ContainerGroupProperties.Properties.RestartPolicy != nil { + objectMap["restartPolicy"] = cg.ContainerGroupProperties.Properties.RestartPolicy } - if cg.ContainerGroupProperties.IPAddress != nil { - objectMap["ipAddress"] = cg.ContainerGroupProperties.IPAddress + if cg.ContainerGroupProperties.Properties.IPAddress != nil { + objectMap["ipAddress"] = cg.ContainerGroupProperties.Properties.IPAddress } - if cg.ContainerGroupProperties.OsType != "" { - objectMap["osType"] = cg.ContainerGroupProperties.OsType + if cg.ContainerGroupProperties.Properties.OSType != nil { + objectMap["osType"] = cg.ContainerGroupProperties.Properties.OSType } - if cg.ContainerGroupProperties.Volumes != nil { - objectMap["volumes"] = cg.ContainerGroupProperties.Volumes + if cg.ContainerGroupProperties.Properties.Volumes != nil { + objectMap["volumes"] = cg.ContainerGroupProperties.Properties.Volumes } - if cg.ContainerGroupProperties.Diagnostics != nil { - objectMap["diagnostics"] = cg.ContainerGroupProperties.Diagnostics + if cg.ContainerGroupProperties.Properties.Diagnostics != nil { + objectMap["diagnostics"] = cg.ContainerGroupProperties.Properties.Diagnostics } - if cg.ContainerGroupProperties.SubnetIds != nil { - objectMap["subnetIds"] = cg.ContainerGroupProperties.SubnetIds + if cg.ContainerGroupProperties.Properties.SubnetIDs != nil { + objectMap["subnetIds"] = cg.ContainerGroupProperties.Properties.SubnetIDs } - if cg.ContainerGroupProperties.DNSConfig != nil { - objectMap["dnsConfig"] = cg.ContainerGroupProperties.DNSConfig + if cg.ContainerGroupProperties.Properties.DNSConfig != nil { + objectMap["dnsConfig"] = cg.ContainerGroupProperties.Properties.DNSConfig } - if cg.ContainerGroupProperties.Sku != "" { - objectMap["sku"] = cg.ContainerGroupProperties.Sku + if cg.ContainerGroupProperties.Properties.SKU != nil { + objectMap["sku"] = cg.ContainerGroupProperties.Properties.SKU } - if cg.ContainerGroupProperties.EncryptionProperties != nil { - objectMap["encryptionProperties"] = cg.ContainerGroupProperties.EncryptionProperties + if cg.ContainerGroupProperties.Properties.EncryptionProperties != nil { + objectMap["encryptionProperties"] = cg.ContainerGroupProperties.Properties.EncryptionProperties } - if cg.ContainerGroupProperties.InitContainers != nil { - objectMap["initContainers"] = cg.ContainerGroupProperties.InitContainers + if cg.ContainerGroupProperties.Properties.InitContainers != nil { + objectMap["initContainers"] = cg.ContainerGroupProperties.Properties.InitContainers } if cg.Extensions != nil { objectMap["extensions"] = cg.Extensions diff --git a/pkg/client/client_apis.go b/pkg/client/client_apis.go index b2cee11e..1c419f70 100644 --- a/pkg/client/client_apis.go +++ b/pkg/client/client_apis.go @@ -2,124 +2,140 @@ package client import ( "context" - "encoding/json" "fmt" "net/http" "os" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" - azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" "github.com/pkg/errors" "github.com/virtual-kubelet/azure-aci/pkg/auth" "github.com/virtual-kubelet/azure-aci/pkg/validation" - "github.com/virtual-kubelet/virtual-kubelet/errdefs" "github.com/virtual-kubelet/virtual-kubelet/log" "github.com/virtual-kubelet/virtual-kubelet/node/api" "github.com/virtual-kubelet/virtual-kubelet/trace" - "k8s.io/client-go/util/retry" ) type AzClientsInterface interface { ContainerGroupGetter CreateContainerGroup(ctx context.Context, resourceGroup, podNS, podName string, cg *ContainerGroupWrapper) error - GetContainerGroupInfo(ctx context.Context, resourceGroup, namespace, name, nodeName string) (*azaci.ContainerGroup, error) - GetContainerGroupListResult(ctx context.Context, resourceGroup string) (*[]azaci.ContainerGroup, error) - ListCapabilities(ctx context.Context, region string) (*[]azaci.Capabilities, error) + GetContainerGroupInfo(ctx context.Context, resourceGroup, namespace, name, nodeName string) (*azaciv2.ContainerGroup, error) + GetContainerGroupListResult(ctx context.Context, resourceGroup string) ([]*azaciv2.ContainerGroup, error) + ListCapabilities(ctx context.Context, region string) ([]*azaciv2.Capabilities, error) DeleteContainerGroup(ctx context.Context, resourceGroup, cgName string) error ListLogs(ctx context.Context, resourceGroup, cgName, containerName string, opts api.ContainerLogOpts) (*string, error) - ExecuteContainerCommand(ctx context.Context, resourceGroup, cgName, containerName string, containerReq azaci.ContainerExecRequest) (*azaci.ContainerExecResponse, error) + ExecuteContainerCommand(ctx context.Context, resourceGroup, cgName, containerName string, containerReq azaciv2.ContainerExecRequest) (*azaciv2.ContainerExecResponse, error) } type AzClientsAPIs struct { - ContainersClient azaci.ContainersClient - ContainerGroupClient ContainerGroupsClientWrapper - LocationClient azaci.LocationClient + ContainersClient *azaciv2.ContainersClient + ContainerGroupClient *azaciv2.ContainerGroupsClient + LocationClient *azaciv2.LocationClient } -func NewAzClientsAPIs(ctx context.Context, azConfig auth.Config) *AzClientsAPIs { +func NewAzClientsAPIs(ctx context.Context, azConfig auth.Config) (*AzClientsAPIs, error) { + logger := log.G(ctx).WithField("method", "NewAzClientsAPIs") + ctx, span := trace.StartSpan(ctx, "client.NewAzClientsAPIs") + defer span.End() + obj := AzClientsAPIs{} - cClient := azaci.NewContainersClientWithBaseURI(azConfig.Cloud.Services[cloud.ResourceManager].Endpoint, azConfig.AuthConfig.SubscriptionID) - cClient.Authorizer = azConfig.Authorizer - //needed for metrics - cClient.Client.Authorizer = azConfig.Authorizer - obj.ContainersClient = cClient + logger.Debug("getting azure credential") - cgClient := ContainerGroupsClientWrapper{CGClient: azaci.NewContainerGroupsClientWithBaseURI(azConfig.Cloud.Services[cloud.ResourceManager].Endpoint, azConfig.AuthConfig.SubscriptionID)} - cgClient.CGClient.Authorizer = azConfig.Authorizer - obj.ContainerGroupClient = cgClient + var err error + var credential azcore.TokenCredential + isUserIdentity := len(azConfig.AuthConfig.ClientID) == 0 - lClient := azaci.NewLocationClientWithBaseURI(azConfig.Cloud.Services[cloud.ResourceManager].Endpoint, azConfig.AuthConfig.SubscriptionID) - lClient.Authorizer = azConfig.Authorizer - //needed for metadata - lClient.Client.Authorizer = azConfig.Authorizer - obj.LocationClient = lClient + if isUserIdentity { + credential, err = azConfig.GetMSICredential(ctx) + } else { + credential, err = azConfig.GetSPCredential(ctx) + } + if err != nil { + return nil, errors.Wrap(err, "an error has occurred while creating getting credential ") + } - obj.setUserAgent(ctx) + logger.Debug("setting aci user agent") + userAgent := os.Getenv("ACI_EXTRA_USER_AGENT") + options := arm.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Cloud: azConfig.Cloud, + Telemetry: policy.TelemetryOptions{ + ApplicationID: userAgent, + }, + }, + } - return &obj -} + logger.Debug("initializing aci clients") + cClient, err := azaciv2.NewContainersClient(azConfig.AuthConfig.SubscriptionID, credential, &options) + if err != nil { + return nil, errors.Wrap(err, "failed to create container client ") + } -func (a *AzClientsAPIs) setUserAgent(ctx context.Context) { - ua := os.Getenv("ACI_EXTRA_USER_AGENT") - if ua != "" { - err := a.ContainersClient.AddToUserAgent(ua) - if err != nil { - log.G(ctx).Warnf("an error has occurred while setting user agent to ContainersClient", err) - return - } - err = a.ContainerGroupClient.CGClient.AddToUserAgent(ua) - if err != nil { - log.G(ctx).Warnf("an error has occurred while setting user agent to ContainerGroupClient", err) - return - } - err = a.LocationClient.AddToUserAgent(ua) - if err != nil { - log.G(ctx).Warnf("an error has occurred while setting user agent to LocationClient", err) - return - } + cgClient, err := azaciv2.NewContainerGroupsClient(azConfig.AuthConfig.SubscriptionID, credential, &options) + if err != nil { + return nil, errors.Wrap(err, "failed to create container group client ") } + + lClient, err := azaciv2.NewLocationClient(azConfig.AuthConfig.SubscriptionID, credential, &options) + if err != nil { + return nil, errors.Wrap(err, "failed to create location client ") + } + + obj.ContainersClient = cClient + obj.ContainerGroupClient = cgClient + obj.LocationClient = lClient + + logger.Debug("aci clients have been initialized successfully") + return &obj, nil } -func (a *AzClientsAPIs) GetContainerGroup(ctx context.Context, resourceGroup, containerGroupName string) (*ContainerGroupWrapper, error) { +func (a *AzClientsAPIs) GetContainerGroup(ctx context.Context, resourceGroup, containerGroupName string) (*azaciv2.ContainerGroup, error) { logger := log.G(ctx).WithField("method", "GetContainerGroup") - ctx, span := trace.StartSpan(ctx, "aci.GetContainerGroup") + ctx, span := trace.StartSpan(ctx, "client.GetContainerGroup") defer span.End() - getPreparer, err := a.ContainerGroupClient.CGClient.GetPreparer(ctx, resourceGroup, containerGroupName) - if err != nil { - return nil, err - } - result, err := a.ContainerGroupClient.CGClient.GetSender(getPreparer) + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) + + result, err := a.ContainerGroupClient.Get(ctxWithResp, resourceGroup, containerGroupName, nil) if err != nil { + if rawResponse.StatusCode == http.StatusNotFound { + logger.Errorf("failed to query Container Group %s, not found", containerGroupName) + return nil, err + } return nil, err } - if result.Body == nil { - return nil, errors.New("get container group returned an empty body in the response") - } - - logger.Infof("GetContainerGroup status code: %d", result.StatusCode) - if result.StatusCode != http.StatusOK { - return nil, errors.Errorf("get container group failed with status code %d", result.StatusCode) - } - var cgw ContainerGroupWrapper - if err := json.NewDecoder(result.Body).Decode(&cgw); err != nil { - return nil, fmt.Errorf("decoding get container group response body failed: %v", err) - } - return &cgw, nil + return &result.ContainerGroup, nil } func (a *AzClientsAPIs) CreateContainerGroup(ctx context.Context, resourceGroup, podNS, podName string, cg *ContainerGroupWrapper) error { logger := log.G(ctx).WithField("method", "CreateContainerGroup") - ctx, span := trace.StartSpan(ctx, "aci.CreateContainerGroup") + ctx, span := trace.StartSpan(ctx, "client.CreateContainerGroup") defer span.End() - cgName := containerGroupName(podNS, podName) - cg.Name = &cgName - logger.Infof("creating container group with name: %s", *cg.Name) - err := a.ContainerGroupClient.CreateCG(ctx, resourceGroup, *cg) + + containerGroup := azaciv2.ContainerGroup{ + Properties: cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties, + Name: &cgName, + Type: cg.Type, + Identity: cg.Identity, + Location: cg.Location, + Tags: cg.Tags, + ID: cg.ID, + } + + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) + + logger.Infof("creating container group with name: %s", cgName) + _, err := a.ContainerGroupClient.BeginCreateOrUpdate(ctxWithResp, resourceGroup, cgName, containerGroup, nil) if err != nil { + logger.Errorf("an error has occurred while creating container group %s, status code %d", cgName, rawResponse.StatusCode) return err } @@ -127,81 +143,93 @@ func (a *AzClientsAPIs) CreateContainerGroup(ctx context.Context, resourceGroup, } // GetContainerGroupInfo returns a container group from ACI. -func (a *AzClientsAPIs) GetContainerGroupInfo(ctx context.Context, resourceGroup, namespace, name, nodeName string) (*azaci.ContainerGroup, error) { - ctx, span := trace.StartSpan(ctx, "aci.GetContainerGroupInfo") +func (a *AzClientsAPIs) GetContainerGroupInfo(ctx context.Context, resourceGroup, namespace, name, nodeName string) (*azaciv2.ContainerGroup, error) { + logger := log.G(ctx).WithField("method", "GetContainerGroupInfo") + ctx, span := trace.StartSpan(ctx, "client.GetContainerGroupInfo") defer span.End() + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) cgName := containerGroupName(namespace, name) - cg, err := a.ContainerGroupClient.CGClient.Get(ctx, resourceGroup, cgName) + response, err := a.ContainerGroupClient.Get(ctxWithResp, resourceGroup, cgName, nil) if err != nil { - if cg.StatusCode == http.StatusNotFound { - return nil, errdefs.NotFound(fmt.Sprintf("container group %s is not found", name)) - } + logger.Errorf("an error has occurred while getting container group info %s, status code %d", cgName, rawResponse.StatusCode) return nil, err } - err = validation.ValidateContainerGroup(&cg) + err = validation.ValidateContainerGroup(ctx, &response.ContainerGroup) + logger.Debugf("container group %s has missing fields. retrying the validation...", cgName) if err != nil { return nil, err } - if *cg.Tags["NodeName"] != nodeName { - return nil, errors.Wrapf(err, "container group %s found with mismatching node", name) + + if nodeName != "" && *response.Tags["NodeName"] != nodeName { + return nil, errors.Wrapf(err, "container group %s found with mismatching node", cgName) } - return &cg, nil + return &response.ContainerGroup, nil } -func (a *AzClientsAPIs) GetContainerGroupListResult(ctx context.Context, resourceGroup string) (*[]azaci.ContainerGroup, error) { - ctx, span := trace.StartSpan(ctx, "aci.GetContainerGroupListResult") +func (a *AzClientsAPIs) GetContainerGroupListResult(ctx context.Context, resourceGroup string) ([]*azaciv2.ContainerGroup, error) { + logger := log.G(ctx).WithField("method", "GetContainerGroupListResult") + ctx, span := trace.StartSpan(ctx, "client.GetContainerGroupListResult") defer span.End() - cgs, err := a.ContainerGroupClient.CGClient.ListByResourceGroup(ctx, resourceGroup) - if err != nil { - return nil, err - } + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) + + pager := a.ContainerGroupClient.NewListByResourceGroupPager(resourceGroup, nil) - list := cgs.Values() - return &list, nil + var cgList []*azaciv2.ContainerGroup + for pager.More() { + page, err := pager.NextPage(ctxWithResp) + if err != nil { + logger.Errorf("an error has occurred while getting list of container groups, status code %d", rawResponse.StatusCode) + return nil, err + } + cgList = append(cgList, page.Value...) + } + return cgList, nil } -func (a *AzClientsAPIs) ListCapabilities(ctx context.Context, region string) (*[]azaci.Capabilities, error) { +func (a *AzClientsAPIs) ListCapabilities(ctx context.Context, region string) ([]*azaciv2.Capabilities, error) { logger := log.G(ctx).WithField("method", "ListCapabilities") - ctx, span := trace.StartSpan(ctx, "aci.ListCapabilities") + ctx, span := trace.StartSpan(ctx, "client.ListCapabilities") defer span.End() - capabilities, err := a.LocationClient.ListCapabilitiesComplete(ctx, region) + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) - if err != nil { - return nil, errors.Wrapf(err, "Unable to fetch the ACI capabilities for the location %s, skipping GPU availability check. GPU capacity will be disabled", region) - } + pager := a.LocationClient.NewListCapabilitiesPager(region, nil) - logger.Infof("ListCapabilitiesComplete status code: %d", capabilities.Response().StatusCode) - if capabilities.Response().StatusCode != http.StatusOK { - logger.Warn("Unable to fetch the ACI capabilities for the location %s, skipping GPU availability check. GPU capacity will be disabled", region) - return nil, nil + var capList []*azaciv2.Capabilities + for pager.More() { + page, err := pager.NextPage(ctxWithResp) + if err != nil { + return nil, errors.Wrapf(err, "Unable to fetch the ACI capabilities for the location %s, skipping GPU availability check. GPU capacity will be disabled", region) + } + capList = append(capList, page.Value...) } - result := capabilities.Response().Value - if result == nil { + if capList == nil { logger.Warn("ACI GPU capacity is not enabled. GPU capacity will be disabled") return nil, nil } - return result, nil + return capList, nil } func (a *AzClientsAPIs) DeleteContainerGroup(ctx context.Context, resourceGroup, cgName string) error { logger := log.G(ctx).WithField("method", "DeleteContainerGroup") - ctx, span := trace.StartSpan(ctx, "aci.DeleteContainerGroup") + ctx, span := trace.StartSpan(ctx, "client.DeleteContainerGroup") defer span.End() - deleteFuture, err := a.ContainerGroupClient.CGClient.Delete(ctx, resourceGroup, cgName) - if err != nil { - logger.Errorf("failed to delete container group %v", cgName) - return err - } - err = deleteFuture.WaitForCompletionRef(ctx, a.ContainerGroupClient.CGClient.Client) + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) + + _, err := a.ContainerGroupClient.BeginDelete(ctxWithResp, resourceGroup, cgName, nil) if err != nil { + logger.Errorf("failed to delete container group %s, status code %d", cgName, rawResponse.StatusCode) return err } @@ -211,9 +239,12 @@ func (a *AzClientsAPIs) DeleteContainerGroup(ctx context.Context, resourceGroup, func (a *AzClientsAPIs) ListLogs(ctx context.Context, resourceGroup, cgName, containerName string, opts api.ContainerLogOpts) (*string, error) { logger := log.G(ctx).WithField("method", "ListLogs") - ctx, span := trace.StartSpan(ctx, "aci.ListLogs") + ctx, span := trace.StartSpan(ctx, "client.ListLogs") defer span.End() + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) + enableTimestamp := true // tail should be > 0, otherwise, set to nil @@ -224,39 +255,37 @@ func (a *AzClientsAPIs) ListLogs(ctx context.Context, resourceGroup, cgName, con } else { logTail = &tail } - var err error - var result azaci.Logs - err = retry.OnError(retry.DefaultBackoff, - func(err error) bool { - return ctx.Err() == nil - }, func() error { - result, err = a.ContainersClient.ListLogs(ctx, resourceGroup, cgName, containerName, logTail, &enableTimestamp) - if err != nil { - logger.Debug("error getting container logs, name: %s , container group: %s, retrying", containerName, cgName) - return err - } - return nil - }) + + options := azaciv2.ContainersClientListLogsOptions{ + Tail: logTail, + Timestamps: &enableTimestamp, + } + + response, err := a.ContainersClient.ListLogs(ctxWithResp, resourceGroup, cgName, containerName, &options) if err != nil { - logger.Errorf("error getting container logs, name: %s , container group: %s", containerName, cgName) + logger.Errorf("error getting container logs, name: %s , container group: %s, status code %d", containerName, cgName, rawResponse.StatusCode) return nil, err } - logger.Infof("ListLogs status code: %d", result.StatusCode) - return result.Content, nil + return response.Content, nil } -func (a *AzClientsAPIs) ExecuteContainerCommand(ctx context.Context, resourceGroup, cgName, containerName string, containerReq azaci.ContainerExecRequest) (*azaci.ContainerExecResponse, error) { +func (a *AzClientsAPIs) ExecuteContainerCommand(ctx context.Context, resourceGroup, cgName, containerName string, containerReq azaciv2.ContainerExecRequest) (*azaciv2.ContainerExecResponse, error) { logger := log.G(ctx).WithField("method", "ExecuteContainerCommand") - ctx, span := trace.StartSpan(ctx, "aci.ExecuteContainerCommand") + ctx, span := trace.StartSpan(ctx, "client.ExecuteContainerCommand") defer span.End() - result, err := a.ContainersClient.ExecuteCommand(ctx, resourceGroup, cgName, containerName, containerReq) + var rawResponse *http.Response + ctxWithResp := runtime.WithCaptureResponse(ctx, &rawResponse) + + result, err := a.ContainersClient.ExecuteCommand(ctxWithResp, resourceGroup, cgName, containerName, containerReq, nil) if err != nil { + logger.Errorf("an error has occurred while executing command for container group %s, status code %d", cgName, rawResponse.StatusCode) return nil, err } - logger.Infof("ExecuteContainerCommand status code: %d", result.StatusCode) - return &result, nil + + logger.Debug("ExecuteContainerCommand is successful") + return &result.ContainerExecResponse, nil } func containerGroupName(podNS, podName string) string { diff --git a/pkg/client/extensions.go b/pkg/client/extensions.go index 5d8e99e7..19805491 100644 --- a/pkg/client/extensions.go +++ b/pkg/client/extensions.go @@ -8,52 +8,30 @@ import ( "io/ioutil" "os" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1" ) -// Extension is the container group extension -type Extension struct { - Name string `json:"name"` - Properties *ExtensionProperties `json:"properties"` -} - -// ExtensionProperties is the properties for extension -type ExtensionProperties struct { - Type ExtensionType `json:"extensionType"` - Version ExtensionVersion `json:"version"` - Settings map[string]string `json:"settings,omitempty"` - ProtectedSettings map[string]string `json:"protectedSettings,omitempty"` -} +var ( + // Supported extension types -// ExtensionType is an enum type for defining supported extension types -type ExtensionType string + ExtensionTypeKubeProxy = "kube-proxy" + ExtensionTypeRealtimeMetrics = "realtime-metrics" -// Supported extension types -const ( - ExtensionTypeKubeProxy ExtensionType = "kube-proxy" - ExtensionTypeRealtimeMetrics ExtensionType = "realtime-metrics" -) - -// ExtensionVersion is an enum type for defining supported extension versions -type ExtensionVersion string - -const ( // ExtensionVersion_1 Supported extension version. - ExtensionVersion_1 ExtensionVersion = "1.0" -) + ExtensionVersion_1 = "1.0" -// Supported kube-proxy extension constants -const ( - KubeProxyExtensionSettingClusterCIDR string = "clusterCidr" - KubeProxyExtensionSettingKubeVersion string = "kubeVersion" - KubeProxyExtensionSettingKubeConfig string = "kubeConfig" - KubeProxyExtensionKubeVersion string = "v1.9.10" + // Supported kube-proxy extension constants + KubeProxyExtensionSettingClusterCIDR = "clusterCidr" + KubeProxyExtensionSettingKubeVersion = "kubeVersion" + KubeProxyExtensionSettingKubeConfig = "kubeConfig" + KubeProxyExtensionKubeVersion = "v1.9.10" ) -// GetKubeProxyExtension gets the kubeproxy extension -func GetKubeProxyExtension(secretPath, masterURI, clusterCIDR string) (*Extension, error) { +// GetKubeProxyExtension gets the kubeProxy extension +func GetKubeProxyExtension(secretPath, masterURI, clusterCIDR string) (*azaciv2.DeploymentExtensionSpec, error) { name := "virtual-kubelet" var certAuthData []byte var authInfo *clientcmdapi.AuthInfo @@ -136,11 +114,11 @@ func GetKubeProxyExtension(secretPath, masterURI, clusterCIDR string) (*Extensio return nil, fmt.Errorf("failed to encode the kubeconfig: %v", err) } - extension := Extension{ - Name: "kube-proxy", - Properties: &ExtensionProperties{ - Type: ExtensionTypeKubeProxy, - Version: ExtensionVersion_1, + extension := azaciv2.DeploymentExtensionSpec{ + Name: &ExtensionTypeKubeProxy, + Properties: &azaciv2.DeploymentExtensionSpecProperties{ + ExtensionType: &ExtensionTypeKubeProxy, + Version: &ExtensionVersion_1, Settings: map[string]string{ KubeProxyExtensionSettingClusterCIDR: clusterCIDR, KubeProxyExtensionSettingKubeVersion: KubeProxyExtensionKubeVersion, @@ -170,12 +148,12 @@ func getKubeconfigAuthInfo(authInfos map[string]*clientcmdapi.AuthInfo) *clientc } // GetRealtimeMetricsExtension gets the realtime extension -func GetRealtimeMetricsExtension() *Extension { - extension := Extension{ - Name: "vk-realtime-metrics", - Properties: &ExtensionProperties{ - Type: ExtensionTypeRealtimeMetrics, - Version: ExtensionVersion_1, +func GetRealtimeMetricsExtension() *azaciv2.DeploymentExtensionSpec { + extension := azaciv2.DeploymentExtensionSpec{ + Name: &ExtensionTypeRealtimeMetrics, + Properties: &azaciv2.DeploymentExtensionSpecProperties{ + ExtensionType: &ExtensionTypeRealtimeMetrics, + Version: &ExtensionVersion_1, Settings: map[string]string{}, ProtectedSettings: map[string]string{}, }, diff --git a/pkg/client/interfaces.go b/pkg/client/interfaces.go index 7ea708d9..3430d769 100644 --- a/pkg/client/interfaces.go +++ b/pkg/client/interfaces.go @@ -3,6 +3,7 @@ package client import ( "context" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" stats "github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1" v1 "k8s.io/api/core/v1" ) @@ -14,7 +15,7 @@ type PodGetter interface { // ContainerGroupGetter package dependency: query the Container Group information type ContainerGroupGetter interface { - GetContainerGroup(ctx context.Context, resourceGroup, containerGroupName string) (*ContainerGroupWrapper, error) + GetContainerGroup(ctx context.Context, resourceGroup, containerGroupName string) (*azaciv2.ContainerGroup, error) } /* diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 6ffc8548..5c805915 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -3,10 +3,10 @@ package metrics import ( "context" "fmt" - "net/http" "sync" "time" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" "github.com/patrickmn/go-cache" "github.com/pkg/errors" "github.com/virtual-kubelet/azure-aci/pkg/client" @@ -62,7 +62,7 @@ func (p *ACIPodMetricsProvider) GetStatsSummary(ctx context.Context) (summary *s pods := p.podGetter.GetPods() var errGroup errgroup.Group - chResult := make(chan stats.PodStats, len(pods)) + chResult := make(chan *stats.PodStats, len(pods)) sema := make(chan struct{}, 10) for _, pod := range pods { @@ -95,8 +95,13 @@ func (p *ACIPodMetricsProvider) GetStatsSummary(ctx context.Context) (summary *s span.SetStatus(err) return errors.Wrapf(err, "error fetching metrics for pods '%s'", pod.Name) } + if podMetrics == nil { + err := fmt.Errorf("error fetching metrics for pods '%s'. cannot retrieve the pod status", pod.Name) + span.SetStatus(err) + return err + } - chResult <- *podMetrics + chResult <- podMetrics return nil }) } @@ -115,7 +120,7 @@ func (p *ACIPodMetricsProvider) GetStatsSummary(ctx context.Context) (summary *s s.Pods = make([]stats.PodStats, 0, len(chResult)) for stat := range chResult { - s.Pods = append(s.Pods, stat) + s.Pods = append(s.Pods, *stat) } return &s, nil @@ -152,8 +157,9 @@ func (decider *podStatsGetterDecider) GetPodStats(ctx context.Context, pod *v1.P } useRealTime := false - for _, extension := range aciCG.ContainerGroupPropertiesWrapper.Extensions { - if extension.Properties.Type == client.ExtensionTypeRealtimeMetrics { + for _, extension := range aciCG.Properties.Extensions { + if extension.Properties.ExtensionType != nil && + *extension.Properties.ExtensionType == client.ExtensionTypeRealtimeMetrics { useRealTime = true } } @@ -167,19 +173,16 @@ func (decider *podStatsGetterDecider) GetPodStats(ctx context.Context, pod *v1.P } } -func (decider *podStatsGetterDecider) getContainerGroupFromPod(ctx context.Context, pod *v1.Pod) (*client.ContainerGroupWrapper, error) { +func (decider *podStatsGetterDecider) getContainerGroupFromPod(ctx context.Context, pod *v1.Pod) (*azaciv2.ContainerGroup, error) { cgName := containerGroupName(pod.Namespace, pod.Name) cacheKey := string(pod.UID) aciContainerGroup, found := decider.cache.Get(cacheKey) if found { - return aciContainerGroup.(*client.ContainerGroupWrapper), nil + return aciContainerGroup.(*azaciv2.ContainerGroup), nil } aciCG, err := decider.aciCGGetter.GetContainerGroup(ctx, decider.rgName, cgName) if err != nil { - if aciCG != nil && aciCG.StatusCode == http.StatusNotFound { - return nil, errors.Wrapf(err, "failed to query Container Group %s, not found it", cgName) - } - return nil, errors.Wrapf(err, "failed to query Container Group %s", cgName) + return nil, err } decider.cache.Set(cacheKey, aciCG, cache.DefaultExpiration) return aciCG, nil diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index 737b170e..b70486a5 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/virtual-kubelet/azure-aci/pkg/client" @@ -75,7 +76,7 @@ func TestPodStatsGetterDecider(t *testing.T) { // Times(1) here because we expect the Container Group be cached mockedAciCgGetter.EXPECT().GetContainerGroup( - gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeContainerGroupWrapper(), nil).Times(1) + gomock.Any(), gomock.Any(), gomock.Any()).Return(fakeContainerGroup(), nil).Times(1) mockedRealtime := NewMockpodStatsGetter(ctrl) mockedRealtime.EXPECT().GetPodStats(gomock.Any(), gomock.Any()).Return(fakePodStatus("pod-1", 0), nil).Times(1) @@ -119,14 +120,14 @@ func fakePodStatus(podName string, cpu uint64) *stats.PodStats { } } -func fakeContainerGroupWrapper() *client.ContainerGroupWrapper { - return &client.ContainerGroupWrapper{ - ContainerGroupPropertiesWrapper: &client.ContainerGroupPropertiesWrapper{ - Extensions: []*client.Extension{ +func fakeContainerGroup() *azaciv2.ContainerGroup { + return &azaciv2.ContainerGroup{ + Properties: &azaciv2.ContainerGroupPropertiesProperties{ + Extensions: []*azaciv2.DeploymentExtensionSpec{ { - Properties: &client.ExtensionProperties{ - Type: client.ExtensionTypeKubeProxy, - Version: client.ExtensionVersion_1, + Properties: &azaciv2.DeploymentExtensionSpecProperties{ + ExtensionType: &client.ExtensionTypeKubeProxy, + Version: &client.ExtensionVersion_1, Settings: map[string]string{ client.KubeProxyExtensionSettingClusterCIDR: "10.240.0.0/16", client.KubeProxyExtensionSettingKubeVersion: client.KubeProxyExtensionKubeVersion, @@ -135,9 +136,9 @@ func fakeContainerGroupWrapper() *client.ContainerGroupWrapper { }, }, { - Properties: &client.ExtensionProperties{ - Type: client.ExtensionTypeRealtimeMetrics, - Version: client.ExtensionVersion_1, + Properties: &azaciv2.DeploymentExtensionSpecProperties{ + ExtensionType: &client.ExtensionTypeRealtimeMetrics, + Version: &client.ExtensionVersion_1, Settings: map[string]string{}, ProtectedSettings: map[string]string{}, }, diff --git a/pkg/metrics/mock_metrics_test.go b/pkg/metrics/mock_metrics_test.go index f9a95d5d..0b1b6361 100644 --- a/pkg/metrics/mock_metrics_test.go +++ b/pkg/metrics/mock_metrics_test.go @@ -8,6 +8,7 @@ import ( context "context" reflect "reflect" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" gomock "github.com/golang/mock/gomock" "github.com/virtual-kubelet/azure-aci/pkg/client" statsv1alpha1 "github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1" @@ -98,10 +99,10 @@ func (m *MockContainerGroupGetter) EXPECT() *MockContainerGroupGetterMockRecorde } // GetContainerGroup mocks base method. -func (m *MockContainerGroupGetter) GetContainerGroup(ctx context.Context, resourceGroup, containerGroupName string) (*client.ContainerGroupWrapper, error) { +func (m *MockContainerGroupGetter) GetContainerGroup(ctx context.Context, resourceGroup, containerGroupName string) (*azaciv2.ContainerGroup, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetContainerGroupInfo", ctx, resourceGroup, containerGroupName) - ret0, _ := ret[0].(*client.ContainerGroupWrapper) + ret0, _ := ret[0].(*azaciv2.ContainerGroup) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/pkg/network/aci_network.go b/pkg/network/aci_network.go index cae5f9fa..cdd5e200 100644 --- a/pkg/network/aci_network.go +++ b/pkg/network/aci_network.go @@ -17,7 +17,7 @@ import ( "github.com/virtual-kubelet/virtual-kubelet/trace" utilvalidation "k8s.io/apimachinery/pkg/util/validation" - azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" aznetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" "github.com/virtual-kubelet/azure-aci/pkg/auth" client2 "github.com/virtual-kubelet/azure-aci/pkg/client" @@ -183,27 +183,28 @@ func (pn *ProviderNetwork) AmendVnetResources(ctx context.Context, cg client2.Co } subnetID := "/subscriptions/" + pn.VnetSubscriptionID + "/resourceGroups/" + pn.VnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + pn.VnetName + "/subnets/" + pn.SubnetName - cgIDList := []azaci.ContainerGroupSubnetID{{ID: &subnetID}} - cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.SubnetIds = &cgIDList + cgIDList := []*azaciv2.ContainerGroupSubnetID{{ID: &subnetID}} + cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.SubnetIDs = cgIDList // windows containers don't support DNS config - if cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.OsType != azaci.OperatingSystemTypesWindows { - cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.DNSConfig = getDNSConfig(ctx, pod, pn.KubeDNSIP, clusterDomain) + if cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.OSType != nil && + *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.OSType != azaciv2.OperatingSystemTypesWindows { + cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.DNSConfig = getDNSConfig(ctx, pod, pn.KubeDNSIP, clusterDomain) } } -func getDNSConfig(ctx context.Context, pod *v1.Pod, kubeDNSIP, clusterDomain string) *azaci.DNSConfiguration { - nameServers := make([]string, 0) +func getDNSConfig(ctx context.Context, pod *v1.Pod, kubeDNSIP, clusterDomain string) *azaciv2.DNSConfiguration { + servers := make([]string, 0) searchDomains := make([]string, 0) if pod.Spec.DNSPolicy == v1.DNSClusterFirst || pod.Spec.DNSPolicy == v1.DNSClusterFirstWithHostNet { - nameServers = append(nameServers, kubeDNSIP) + servers = append(servers, kubeDNSIP) searchDomains = generateSearchesForDNSClusterFirst(pod.Spec.DNSConfig, pod, clusterDomain) } options := make([]string, 0) if pod.Spec.DNSConfig != nil { - nameServers = util.OmitDuplicates(append(nameServers, pod.Spec.DNSConfig.Nameservers...)) + servers = util.OmitDuplicates(append(servers, pod.Spec.DNSConfig.Nameservers...)) searchDomains = util.OmitDuplicates(append(searchDomains, pod.Spec.DNSConfig.Searches...)) for _, option := range pod.Spec.DNSConfig.Options { @@ -215,14 +216,18 @@ func getDNSConfig(ctx context.Context, pod *v1.Pod, kubeDNSIP, clusterDomain str } } - if len(nameServers) == 0 { + if len(servers) == 0 { return nil } - nameServers = formDNSNameserversFitsLimits(ctx, nameServers) + servers = formDNSNameserversFitsLimits(ctx, servers) domain := formDNSSearchFitsLimits(ctx, searchDomains) + nameServers := make([]*string, 0) + for s := range servers { + nameServers = append(nameServers, &servers[s]) + } opt := strings.Join(options, " ") - result := azaci.DNSConfiguration{ - NameServers: &nameServers, + result := azaciv2.DNSConfiguration{ + NameServers: nameServers, SearchDomains: &domain, Options: &opt, } diff --git a/pkg/network/aci_network_test.go b/pkg/network/aci_network_test.go index 24ea1c2d..d238f08e 100644 --- a/pkg/network/aci_network_test.go +++ b/pkg/network/aci_network_test.go @@ -68,7 +68,7 @@ func TestGetDNSConfig(t *testing.T) { aciDNSConfig := getDNSConfig(ctx, testPod, kubeDNSIP, clusterDomain) if tc.kubeDNSIP { - assert.Contains(t, *aciDNSConfig.NameServers, kubeDNSIP, "test [%d]", i) + assert.Contains(t, aciDNSConfig.NameServers, &kubeDNSIP, "test [%d]", i) } if tc.shouldHaveClusterDomain { assert.Contains(t, *aciDNSConfig.SearchDomains, clusterDomain, "test [%d]", i) diff --git a/pkg/provider/aci.go b/pkg/provider/aci.go index 6720ca76..1508c8d4 100644 --- a/pkg/provider/aci.go +++ b/pkg/provider/aci.go @@ -15,13 +15,12 @@ import ( "strings" "time" - azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" "github.com/gorilla/websocket" "github.com/pkg/errors" - "github.com/virtual-kubelet/azure-aci/client/aci" "github.com/virtual-kubelet/azure-aci/pkg/analytics" "github.com/virtual-kubelet/azure-aci/pkg/auth" - client2 "github.com/virtual-kubelet/azure-aci/pkg/client" + "github.com/virtual-kubelet/azure-aci/pkg/client" "github.com/virtual-kubelet/azure-aci/pkg/featureflag" "github.com/virtual-kubelet/azure-aci/pkg/metrics" "github.com/virtual-kubelet/azure-aci/pkg/network" @@ -51,9 +50,6 @@ const ( AzureFileDriverName = "file.csi.azure.com" azureFileStorageAccountName = "azurestorageaccountname" azureFileStorageAccountKey = "azurestorageaccountkey" - - LogAnalyticsMetadataKeyNodeName string = "node-name" - LogAnalyticsMetadataKeyClusterResourceID string = "cluster-resource-id" ) const ( @@ -69,9 +65,9 @@ const ( // ACIProvider implements the virtual-kubelet provider interface and communicates with Azure's ACI APIs. type ACIProvider struct { - azClientsAPIs client2.AzClientsInterface + azClientsAPIs client.AzClientsInterface resourceManager *manager.ResourceManager - containerGroupExtensions []*client2.Extension + containerGroupExtensions []*azaciv2.DeploymentExtensionSpec enabledFeatures *featureflag.FlagIdentifier providernetwork network.ProviderNetwork @@ -83,10 +79,10 @@ type ACIProvider struct { memory string pods string gpu string - gpuSKUs []azaci.GpuSku + gpuSKUs []azaciv2.GpuSKU internalIP string daemonEndpointPort int32 - diagnostics *azaci.ContainerGroupDiagnostics + diagnostics *azaciv2.ContainerGroupDiagnostics clusterDomain string tracker *PodsTracker @@ -164,7 +160,7 @@ func isValidACIRegion(region string) bool { } // NewACIProvider creates a new ACIProvider. -func NewACIProvider(ctx context.Context, config string, azConfig auth.Config, azAPIs client2.AzClientsInterface, rm *manager.ResourceManager, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32, clusterDomain string) (*ACIProvider, error) { +func NewACIProvider(ctx context.Context, config string, azConfig auth.Config, azAPIs client.AzClientsInterface, rm *manager.ResourceManager, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32, clusterDomain string) (*ACIProvider, error) { var p ACIProvider var err error @@ -221,10 +217,10 @@ func NewACIProvider(ctx context.Context, config string, azConfig auth.Config, az if clusterResourceID := os.Getenv("CLUSTER_RESOURCE_ID"); clusterResourceID != "" { if p.diagnostics != nil && p.diagnostics.LogAnalytics != nil { - p.diagnostics.LogAnalytics.LogType = azaci.LogAnalyticsLogTypeContainerInsights + p.diagnostics.LogAnalytics.LogType = &util.LogTypeContainerInsights p.diagnostics.LogAnalytics.Metadata = map[string]*string{ - LogAnalyticsMetadataKeyClusterResourceID: &clusterResourceID, - LogAnalyticsMetadataKeyNodeName: &nodeName, + analytics.LogAnalyticsMetadataKeyClusterResourceID: &clusterResourceID, + analytics.LogAnalyticsMetadataKeyNodeName: &nodeName, } } } @@ -257,7 +253,7 @@ func NewACIProvider(ctx context.Context, config string, azConfig auth.Config, az if p.providernetwork.SubnetName != "" { // windows containers don't support kube-proxy nor realtime metrics - if p.operatingSystem != string(azaci.OperatingSystemTypesWindows) { + if p.operatingSystem != string(azaciv2.OperatingSystemTypesWindows) { err = p.setACIExtensions(ctx) if err != nil { return nil, err @@ -284,15 +280,20 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { defer span.End() ctx = addAzureAttributes(ctx, span, p) - cg := &client2.ContainerGroupWrapper{ - ContainerGroupPropertiesWrapper: &client2.ContainerGroupPropertiesWrapper{ - ContainerGroupProperties: &azaci.ContainerGroupProperties{}, + cg := &client.ContainerGroupWrapper{ + ContainerGroupPropertiesWrapper: &client.ContainerGroupPropertiesWrapper{ + ContainerGroupProperties: &azaciv2.ContainerGroupProperties{ + Properties: &azaciv2.ContainerGroupPropertiesProperties{}, + }, }, } + os := azaciv2.OperatingSystemTypes(p.operatingSystem) + policy := azaciv2.ContainerGroupRestartPolicy(pod.Spec.RestartPolicy) + cg.Location = &p.region - cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.RestartPolicy = azaci.ContainerGroupRestartPolicy(pod.Spec.RestartPolicy) - cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.OsType = azaci.OperatingSystemTypes(p.operatingSystem) + cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.RestartPolicy = &policy + cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.OSType = &os // get containers containers, err := p.getContainers(pod) @@ -317,40 +318,40 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { if err != nil { return err } - cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.InitContainers = &initContainers + cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.InitContainers = initContainers } // assign all the things - cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers = containers - cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes = &volumes - cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.ImageRegistryCredentials = creds - cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Diagnostics = p.getDiagnostics(pod) + cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers = containers + cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Volumes = volumes + cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.ImageRegistryCredentials = creds + cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Diagnostics = p.getDiagnostics(pod) filterWindowsServiceAccountSecretVolume(ctx, p.operatingSystem, cg) // create ipaddress if containerPort is used count := 0 - for _, container := range *containers { - count = count + len(*container.Ports) - } - ports := make([]azaci.Port, 0, count) - for c := range *containers { - containerPorts := ((*containers)[c]).Ports - for p := range *containerPorts { - ports = append(ports, azaci.Port{ - Port: (*containerPorts)[p].Port, - Protocol: azaci.ContainerGroupNetworkProtocolTCP, + for _, container := range containers { + count = count + len(container.Properties.Ports) + } + ports := make([]*azaciv2.Port, 0, count) + for c := range containers { + containerPorts := containers[c].Properties.Ports + for p := range containerPorts { + ports = append(ports, &azaciv2.Port{ + Port: containerPorts[p].Port, + Protocol: &util.ContainerGroupNetworkProtocolTCP, }) } } if len(ports) > 0 && p.providernetwork.SubnetName == "" { - cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.IPAddress = &azaci.IPAddress{ - Ports: &ports, - Type: azaci.ContainerGroupIPAddressTypePublic, + cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.IPAddress = &azaciv2.IPAddress{ + Ports: ports, + Type: &util.ContainerGroupIPAddressTypePublic, } if dnsNameLabel := pod.Annotations[virtualKubeletDNSNameLabel]; dnsNameLabel != "" { - cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.IPAddress.DNSNameLabel = &dnsNameLabel + cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.IPAddress.DNSNameLabel = &dnsNameLabel } } @@ -368,8 +369,9 @@ func (p *ACIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error { p.providernetwork.AmendVnetResources(ctx, *cg, pod, p.clusterDomain) // windows containers don't support kube-proxy nor realtime metrics - if cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.OsType != azaci.OperatingSystemTypesWindows { - cg.ContainerGroupPropertiesWrapper.Extensions = p.containerGroupExtensions + if cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.OSType != nil && + *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.OSType != azaciv2.OperatingSystemTypesWindows { + cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Extensions = p.containerGroupExtensions } log.G(ctx).Infof("start creating pod %v", pod.Name) @@ -388,7 +390,7 @@ func (p *ACIProvider) setACIExtensions(ctx context.Context) error { clusterCIDR = "10.240.0.0/16" } - kubeExtensions, err := client2.GetKubeProxyExtension(serviceAccountSecretMountPath, masterURI, clusterCIDR) + kubeExtensions, err := client.GetKubeProxyExtension(serviceAccountSecretMountPath, masterURI, clusterCIDR) if err != nil { return fmt.Errorf("error creating kube proxy extension: %v", err) } @@ -397,17 +399,19 @@ func (p *ACIProvider) setACIExtensions(ctx context.Context) error { enableRealTimeMetricsExtension := os.Getenv("ENABLE_REAL_TIME_METRICS") if enableRealTimeMetricsExtension == "true" { - realtimeExtension := client2.GetRealtimeMetricsExtension() + realtimeExtension := client.GetRealtimeMetricsExtension() p.containerGroupExtensions = append(p.containerGroupExtensions, realtimeExtension) } return nil } -func (p *ACIProvider) getDiagnostics(pod *v1.Pod) *azaci.ContainerGroupDiagnostics { - if p.diagnostics != nil && p.diagnostics.LogAnalytics != nil && p.diagnostics.LogAnalytics.LogType == azaci.LogAnalyticsLogTypeContainerInsights { +func (p *ACIProvider) getDiagnostics(pod *v1.Pod) *azaciv2.ContainerGroupDiagnostics { + if p.diagnostics != nil && p.diagnostics.LogAnalytics != nil && + p.diagnostics.LogAnalytics.LogType != nil && + *p.diagnostics.LogAnalytics.LogType == azaciv2.LogAnalyticsLogTypeContainerInsights { d := *p.diagnostics uID := string(pod.ObjectMeta.UID) - d.LogAnalytics.Metadata[aci.LogAnalyticsMetadataKeyPodUUID] = &uID + d.LogAnalytics.Metadata[analytics.LogAnalyticsMetadataKeyPodUUID] = &uID return &d } return p.diagnostics @@ -483,6 +487,7 @@ func (p *ACIProvider) deleteContainerGroup(ctx context.Context, podNS, podName s // GetPod returns a pod by name that is running inside ACI // returns nil if a pod by that name is not found. func (p *ACIProvider) GetPod(ctx context.Context, namespace, name string) (*v1.Pod, error) { + logger := log.G(ctx).WithField("method", "GetPod") ctx, span := trace.StartSpan(ctx, "aci.GetPod") defer span.End() ctx = addAzureAttributes(ctx, span, p) @@ -492,11 +497,13 @@ func (p *ACIProvider) GetPod(ctx context.Context, namespace, name string) (*v1.P return nil, err } - err = validation.ValidateContainerGroup(cg) + err = validation.ValidateContainerGroup(ctx, cg) + logger.Debugf("container group %s has missing fields. retrying the validation...", *cg.Name) if err != nil { return nil, err } - return p.containerGroupToPod(cg) + + return p.containerGroupToPod(ctx, cg) } // GetContainerLogs returns the logs of a pod by name that is running inside ACI. @@ -549,9 +556,9 @@ func (p *ACIProvider) RunInContainer(ctx context.Context, namespace, name, conta cols := int32(60) rows := int32(120) cmdParam := strings.Join(cmd, " ") - req := azaci.ContainerExecRequest{ + req := azaciv2.ContainerExecRequest{ Command: &cmdParam, - TerminalSize: &azaci.ContainerExecRequestTerminalSize{ + TerminalSize: &azaciv2.ContainerExecRequestTerminalSize{ Cols: &cols, Rows: &rows, }, @@ -634,6 +641,7 @@ func (p *ACIProvider) RunInContainer(ctx context.Context, namespace, name, conta // GetPodStatus returns the status of a pod by name that is running inside ACI // returns nil if a pod by that name is not found. func (p *ACIProvider) GetPodStatus(ctx context.Context, namespace, name string) (*v1.PodStatus, error) { + logger := log.G(ctx).WithField("method", "GetPodStatus") ctx, span := trace.StartSpan(ctx, "aci.GetPodStatus") defer span.End() ctx = addAzureAttributes(ctx, span, p) @@ -643,15 +651,18 @@ func (p *ACIProvider) GetPodStatus(ctx context.Context, namespace, name string) return nil, err } - err = validation.ValidateContainerGroup(cg) + err = validation.ValidateContainerGroup(ctx, cg) + logger.Debugf("container group %s has missing fields. retrying the validation...", *(*cg).Name) if err != nil { return nil, err } - return p.getPodStatusFromContainerGroup(cg) + + return p.getPodStatusFromContainerGroup(ctx, cg) } // GetPods returns a list of all pods known to be running within ACI. func (p *ACIProvider) GetPods(ctx context.Context) ([]*v1.Pod, error) { + logger := log.G(ctx).WithField("method", "GetPods") ctx, span := trace.StartSpan(ctx, "aci.GetPods") defer span.End() ctx = addAzureAttributes(ctx, span, p) @@ -660,29 +671,29 @@ func (p *ACIProvider) GetPods(ctx context.Context) ([]*v1.Pod, error) { if err != nil { return nil, err } - if cgs == nil { log.G(ctx).Infof("no container groups found for resource group %s", p.resourceGroup) return nil, nil } - pods := make([]*v1.Pod, 0, len(*cgs)) + pods := make([]*v1.Pod, 0, len(cgs)) - for cgIndex := range *cgs { - validation.ValidateContainerGroup(&(*cgs)[cgIndex]) + for cgIndex := range cgs { + err = validation.ValidateContainerGroup(ctx, cgs[cgIndex]) + logger.Debugf("container group %s has missing fields. retrying the validation...", *cgs[cgIndex].Name) if err != nil { return nil, err } - if (*cgs)[cgIndex].Tags["NodeName"] != &p.nodeName { + if cgs[cgIndex].Tags["NodeName"] != &p.nodeName { continue } - pod, err := p.containerGroupToPod(&(*cgs)[cgIndex]) + pod, err := p.containerGroupToPod(ctx, cgs[cgIndex]) if err != nil { log.G(ctx).WithFields(log.Fields{ - "name": (*cgs)[cgIndex].Name, - "id": (*cgs)[cgIndex].ID, - }).WithError(err).Errorf("error converting container group %s to pod", (*cgs)[cgIndex].Name) + "name": cgs[cgIndex].Name, + "id": cgs[cgIndex].ID, + }).WithError(err).Errorf("error converting container group %s to pod", cgs[cgIndex].Name) continue } @@ -756,12 +767,12 @@ func (p *ACIProvider) Ping(ctx context.Context) error { return nil } -func (p *ACIProvider) getImagePullSecrets(pod *v1.Pod) (*[]azaci.ImageRegistryCredential, error) { - ips := make([]azaci.ImageRegistryCredential, 0, len(pod.Spec.ImagePullSecrets)) +func (p *ACIProvider) getImagePullSecrets(pod *v1.Pod) ([]*azaciv2.ImageRegistryCredential, error) { + ips := make([]*azaciv2.ImageRegistryCredential, 0, len(pod.Spec.ImagePullSecrets)) for _, ref := range pod.Spec.ImagePullSecrets { secret, err := p.resourceManager.GetSecret(ref.Name, pod.Namespace) if err != nil { - return &ips, err + return ips, err } if secret == nil { return nil, fmt.Errorf("error getting image pull secret") @@ -776,14 +787,14 @@ func (p *ACIProvider) getImagePullSecrets(pod *v1.Pod) (*[]azaci.ImageRegistryCr } if err != nil { - return &ips, err + return ips, err } } - return &ips, nil + return ips, nil } -func makeRegistryCredential(server string, authConfig AuthConfig) (*azaci.ImageRegistryCredential, error) { +func makeRegistryCredential(server string, authConfig AuthConfig) (*azaciv2.ImageRegistryCredential, error) { username := authConfig.Username password := authConfig.Password @@ -806,7 +817,7 @@ func makeRegistryCredential(server string, authConfig AuthConfig) (*azaci.ImageR password = parts[1] } - cred := azaci.ImageRegistryCredential{ + cred := azaciv2.ImageRegistryCredential{ Server: &server, Username: &username, Password: &password, @@ -815,12 +826,12 @@ func makeRegistryCredential(server string, authConfig AuthConfig) (*azaci.ImageR return &cred, nil } -func makeRegistryCredentialFromDockerConfig(server string, configEntry DockerConfigEntry) (*azaci.ImageRegistryCredential, error) { +func makeRegistryCredentialFromDockerConfig(server string, configEntry DockerConfigEntry) (*azaciv2.ImageRegistryCredential, error) { if configEntry.Username == "" { return nil, fmt.Errorf("no username present in auth config for server: %s", server) } - cred := azaci.ImageRegistryCredential{ + cred := azaciv2.ImageRegistryCredential{ Server: &server, Username: &configEntry.Username, Password: &configEntry.Password, @@ -829,7 +840,7 @@ func makeRegistryCredentialFromDockerConfig(server string, configEntry DockerCon return &cred, nil } -func readDockerCfgSecret(secret *v1.Secret, ips []azaci.ImageRegistryCredential) ([]azaci.ImageRegistryCredential, error) { +func readDockerCfgSecret(secret *v1.Secret, ips []*azaciv2.ImageRegistryCredential) ([]*azaciv2.ImageRegistryCredential, error) { var err error var authConfigs map[string]AuthConfig repoData, ok := secret.Data[v1.DockerConfigKey] @@ -849,13 +860,13 @@ func readDockerCfgSecret(secret *v1.Secret, ips []azaci.ImageRegistryCredential) return ips, err } - ips = append(ips, *cred) + ips = append(ips, cred) } return ips, err } -func readDockerConfigJSONSecret(secret *v1.Secret, ips []azaci.ImageRegistryCredential) ([]azaci.ImageRegistryCredential, error) { +func readDockerConfigJSONSecret(secret *v1.Secret, ips []*azaciv2.ImageRegistryCredential) ([]*azaciv2.ImageRegistryCredential, error) { var err error repoData, ok := secret.Data[v1.DockerConfigJsonKey] @@ -882,7 +893,7 @@ func readDockerConfigJSONSecret(secret *v1.Secret, ips []azaci.ImageRegistryCred return ips, err } - ips = append(ips, *cred) + ips = append(ips, cred) } return ips, err @@ -897,39 +908,48 @@ func (p *ACIProvider) verifyContainer(container *v1.Container) error { } //this method is used for both initConainers and containers -func (p *ACIProvider) getCommand(container *v1.Container) *[]string { - command := append(container.Command, container.Args...) - return &command +func (p *ACIProvider) getCommand(container v1.Container) []*string { + command := make([]*string, 0) + for c := range container.Command { + command = append(command, &container.Command[c]) + } + + args := make([]*string, 0) + for a := range container.Args { + args = append(args, &container.Args[a]) + } + + return append(command, args...) } //get VolumeMounts declared on Container as []aci.VolumeMount -func (p *ACIProvider) getVolumeMounts(container *v1.Container) *[]azaci.VolumeMount { - volumeMounts := make([]azaci.VolumeMount, 0, len(container.VolumeMounts)) +func (p *ACIProvider) getVolumeMounts(container v1.Container) []*azaciv2.VolumeMount { + volumeMounts := make([]*azaciv2.VolumeMount, 0, len(container.VolumeMounts)) for i := range container.VolumeMounts { - volumeMounts = append(volumeMounts, azaci.VolumeMount{ + volumeMounts = append(volumeMounts, &azaciv2.VolumeMount{ Name: &container.VolumeMounts[i].Name, MountPath: &container.VolumeMounts[i].MountPath, ReadOnly: &container.VolumeMounts[i].ReadOnly, }) } - return &volumeMounts + return volumeMounts } //get EnvironmentVariables declared on Container as []aci.EnvironmentVariable -func (p *ACIProvider) getEnvironmentVariables(container *v1.Container) *[]azaci.EnvironmentVariable { - environmentVariable := make([]azaci.EnvironmentVariable, 0, len(container.Env)) +func (p *ACIProvider) getEnvironmentVariables(container v1.Container) []*azaciv2.EnvironmentVariable { + environmentVariable := make([]*azaciv2.EnvironmentVariable, 0, len(container.Env)) for i := range container.Env { if container.Env[i].Value != "" { envVar := getACIEnvVar(container.Env[i]) environmentVariable = append(environmentVariable, envVar) } } - return &environmentVariable + return environmentVariable } //get InitContainers defined in Pod as []aci.InitContainerDefinition -func (p *ACIProvider) getInitContainers(ctx context.Context, pod *v1.Pod) ([]azaci.InitContainerDefinition, error) { - initContainers := make([]azaci.InitContainerDefinition, 0, len(pod.Spec.InitContainers)) +func (p *ACIProvider) getInitContainers(ctx context.Context, pod *v1.Pod) ([]*azaciv2.InitContainerDefinition, error) { + initContainers := make([]*azaciv2.InitContainerDefinition, 0, len(pod.Spec.InitContainers)) for i, initContainer := range pod.Spec.InitContainers { err := p.verifyContainer(&initContainer) if err != nil { @@ -958,23 +978,23 @@ func (p *ACIProvider) getInitContainers(ctx context.Context, pod *v1.Pod) ([]aza return nil, errdefs.InvalidInput("azure container instances initContainers do not support readinessProbe") } - newInitContainer := azaci.InitContainerDefinition{ + newInitContainer := azaciv2.InitContainerDefinition{ Name: &pod.Spec.InitContainers[i].Name, - InitContainerPropertiesDefinition: &azaci.InitContainerPropertiesDefinition{ + Properties: &azaciv2.InitContainerPropertiesDefinition{ Image: &pod.Spec.InitContainers[i].Image, - Command: p.getCommand(&pod.Spec.InitContainers[i]), - VolumeMounts: p.getVolumeMounts(&pod.Spec.InitContainers[i]), - EnvironmentVariables: p.getEnvironmentVariables(&pod.Spec.InitContainers[i]), + Command: p.getCommand(pod.Spec.InitContainers[i]), + VolumeMounts: p.getVolumeMounts(pod.Spec.InitContainers[i]), + EnvironmentVariables: p.getEnvironmentVariables(pod.Spec.InitContainers[i]), }, } - initContainers = append(initContainers, newInitContainer) + initContainers = append(initContainers, &newInitContainer) } return initContainers, nil } -func (p *ACIProvider) getContainers(pod *v1.Pod) (*[]azaci.Container, error) { - containers := make([]azaci.Container, 0, len(pod.Spec.Containers)) +func (p *ACIProvider) getContainers(pod *v1.Pod) ([]*azaciv2.Container, error) { + containers := make([]*azaciv2.Container, 0, len(pod.Spec.Containers)) podContainers := pod.Spec.Containers for c := range podContainers { @@ -982,46 +1002,41 @@ func (p *ACIProvider) getContainers(pod *v1.Pod) (*[]azaci.Container, error) { if len(podContainers[c].Command) == 0 && len(podContainers[c].Args) > 0 { return nil, errdefs.InvalidInput("ACI does not support providing args without specifying the command. Please supply both command and args to the pod spec.") } - cmd := append(podContainers[c].Command, podContainers[c].Args...) - ports := make([]azaci.ContainerPort, 0, len(podContainers[c].Ports)) - aciContainer := azaci.Container{ + cmd := p.getCommand(podContainers[c]) + ports := make([]*azaciv2.ContainerPort, 0, len(podContainers[c].Ports)) + aciContainer := azaciv2.Container{ Name: &podContainers[c].Name, - ContainerProperties: &azaci.ContainerProperties{ + Properties: &azaciv2.ContainerProperties{ Image: &podContainers[c].Image, - Command: &cmd, - Ports: &ports, + Command: cmd, + Ports: ports, }, } for i := range podContainers[c].Ports { - containerPorts := aciContainer.Ports - containerPortsList := append(*containerPorts, azaci.ContainerPort{ + aciContainer.Properties.Ports = append(aciContainer.Properties.Ports, &azaciv2.ContainerPort{ Port: &podContainers[c].Ports[i].ContainerPort, Protocol: util.GetProtocol(podContainers[c].Ports[i].Protocol), }) - aciContainer.Ports = &containerPortsList } - volMount := make([]azaci.VolumeMount, 0, len(podContainers[c].VolumeMounts)) - aciContainer.VolumeMounts = &volMount + volMount := make([]*azaciv2.VolumeMount, 0, len(podContainers[c].VolumeMounts)) + aciContainer.Properties.VolumeMounts = volMount for v := range podContainers[c].VolumeMounts { - vol := aciContainer.VolumeMounts - volList := append(*vol, azaci.VolumeMount{ + aciContainer.Properties.VolumeMounts = append(aciContainer.Properties.VolumeMounts, &azaciv2.VolumeMount{ Name: &podContainers[c].VolumeMounts[v].Name, MountPath: &podContainers[c].VolumeMounts[v].MountPath, ReadOnly: &podContainers[c].VolumeMounts[v].ReadOnly, }) - aciContainer.VolumeMounts = &volList } - initEnv := make([]azaci.EnvironmentVariable, 0, len(podContainers[c].Env)) - aciContainer.EnvironmentVariables = &initEnv + initEnv := make([]*azaciv2.EnvironmentVariable, 0, len(podContainers[c].Env)) + aciContainer.Properties.EnvironmentVariables = initEnv for _, e := range podContainers[c].Env { - env := aciContainer.EnvironmentVariables if e.Value != "" { envVar := getACIEnvVar(e) - envList := append(*env, envVar) - aciContainer.EnvironmentVariables = &envList + envList := append(aciContainer.Properties.EnvironmentVariables, envVar) + aciContainer.Properties.EnvironmentVariables = envList } } @@ -1043,8 +1058,8 @@ func (p *ACIProvider) getContainers(pod *v1.Pod) (*[]azaci.Container, error) { } } - aciContainer.Resources = &azaci.ResourceRequirements{ - Requests: &azaci.ResourceRequests{ + aciContainer.Properties.Resources = &azaciv2.ResourceRequirements{ + Requests: &azaciv2.ResourceRequests{ CPU: &cpuRequest, MemoryInGB: &memoryRequest, }, @@ -1061,7 +1076,7 @@ func (p *ACIProvider) getContainers(pod *v1.Pod) (*[]azaci.Container, error) { if _, ok := podContainers[c].Resources.Limits[v1.ResourceMemory]; ok { memoryLimit = float64(podContainers[c].Resources.Limits.Memory().Value()/100000000.00) / 10.00 } - aciContainer.Resources.Limits = &azaci.ResourceLimits{ + aciContainer.Properties.Resources.Limits = &azaciv2.ResourceLimits{ CPU: &cpuLimit, MemoryInGB: &memoryLimit, } @@ -1078,13 +1093,13 @@ func (p *ACIProvider) getContainers(pod *v1.Pod) (*[]azaci.Container, error) { count := int32(gpu.Value()) - gpuResource := &azaci.GpuResource{ + gpuResource := &azaciv2.GpuResource{ Count: &count, - Sku: azaci.GpuSku(sku), + SKU: &sku, } - aciContainer.Resources.Requests.Gpu = gpuResource - aciContainer.Resources.Limits.Gpu = gpuResource + aciContainer.Properties.Resources.Requests.Gpu = gpuResource + aciContainer.Properties.Resources.Limits.Gpu = gpuResource } } @@ -1093,7 +1108,7 @@ func (p *ACIProvider) getContainers(pod *v1.Pod) (*[]azaci.Container, error) { if err != nil { return nil, err } - aciContainer.LivenessProbe = probe + aciContainer.Properties.LivenessProbe = probe } if podContainers[c].ReadinessProbe != nil { @@ -1101,15 +1116,15 @@ func (p *ACIProvider) getContainers(pod *v1.Pod) (*[]azaci.Container, error) { if err != nil { return nil, err } - aciContainer.ReadinessProbe = probe + aciContainer.Properties.ReadinessProbe = probe } - containers = append(containers, aciContainer) + containers = append(containers, &aciContainer) } - return &containers, nil + return containers, nil } -func (p *ACIProvider) getGPUSKU(pod *v1.Pod) (azaci.GpuSku, error) { +func (p *ACIProvider) getGPUSKU(pod *v1.Pod) (azaciv2.GpuSKU, error) { if len(p.gpuSKUs) == 0 { return "", fmt.Errorf("the pod requires GPU resource, but ACI doesn't provide GPU enabled container group in region %s", p.region) } @@ -1127,7 +1142,7 @@ func (p *ACIProvider) getGPUSKU(pod *v1.Pod) (azaci.GpuSku, error) { return p.gpuSKUs[0], nil } -func getProbe(probe *v1.Probe, ports []v1.ContainerPort) (*azaci.ContainerProbe, error) { +func getProbe(probe *v1.Probe, ports []v1.ContainerPort) (*azaciv2.ContainerProbe, error) { if probe.Handler.Exec != nil && probe.Handler.HTTPGet != nil { return nil, fmt.Errorf("probe may not specify more than one of \"exec\" and \"httpGet\"") @@ -1140,14 +1155,18 @@ func getProbe(probe *v1.Probe, ports []v1.ContainerPort) (*azaci.ContainerProbe, // Probes have can have an Exec or HTTP Get Handler. // Create those if they exist, then add to the // ContainerProbe struct - var exec *azaci.ContainerExec + var exec *azaciv2.ContainerExec + commands := make([]*string, 0) if probe.Handler.Exec != nil { - exec = &azaci.ContainerExec{ - Command: &(probe.Handler.Exec.Command), + for i := range probe.Handler.Exec.Command { + commands = append(commands, &probe.Handler.Exec.Command[i]) + } + exec = &azaciv2.ContainerExec{ + Command: commands, } } - var httpGET *azaci.ContainerHTTPGet + var httpGET *azaciv2.ContainerHTTPGet if probe.Handler.HTTPGet != nil { var portValue int32 port := probe.Handler.HTTPGet.Port @@ -1167,14 +1186,15 @@ func getProbe(probe *v1.Probe, ports []v1.ContainerPort) (*azaci.ContainerProbe, } } - httpGET = &azaci.ContainerHTTPGet{ + scheme := azaciv2.Scheme(probe.Handler.HTTPGet.Scheme) + httpGET = &azaciv2.ContainerHTTPGet{ Port: &portValue, Path: &probe.Handler.HTTPGet.Path, - Scheme: azaci.Scheme(probe.Handler.HTTPGet.Scheme), + Scheme: &scheme, } } - return &azaci.ContainerProbe{ + return &azaciv2.ContainerProbe{ Exec: exec, HTTPGet: httpGET, InitialDelaySeconds: &probe.InitialDelaySeconds, @@ -1188,20 +1208,20 @@ func getProbe(probe *v1.Probe, ports []v1.ContainerPort) (*azaci.ContainerProbe, // Filters service account secret volume for Windows. // Service account secret volume gets automatically turned on if not specified otherwise. // ACI doesn't support secret volume for Windows, so we need to filter it. -func filterWindowsServiceAccountSecretVolume(ctx context.Context, osType string, cgw *client2.ContainerGroupWrapper) { +func filterWindowsServiceAccountSecretVolume(ctx context.Context, osType string, cgw *client.ContainerGroupWrapper) { if strings.EqualFold(osType, "Windows") { serviceAccountSecretVolumeName := make(map[string]bool) - for index, container := range *cgw.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers { - volumeMounts := make([]azaci.VolumeMount, 0, len(*container.VolumeMounts)) - for _, volumeMount := range *container.VolumeMounts { + for index, container := range cgw.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers { + volumeMounts := make([]*azaciv2.VolumeMount, 0, len(container.Properties.VolumeMounts)) + for _, volumeMount := range container.Properties.VolumeMounts { if !strings.EqualFold(serviceAccountSecretMountPath, *volumeMount.MountPath) { volumeMounts = append(volumeMounts, volumeMount) } else { serviceAccountSecretVolumeName[*volumeMount.Name] = true } } - (*cgw.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers)[index].VolumeMounts = &volumeMounts + cgw.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers[index].Properties.VolumeMounts = volumeMounts } if len(serviceAccountSecretVolumeName) == 0 { @@ -1211,30 +1231,30 @@ func filterWindowsServiceAccountSecretVolume(ctx context.Context, osType string, l := log.G(ctx).WithField("containerGroup", cgw.Name) l.Infof("Ignoring service account secret volumes '%v' for Windows", reflect.ValueOf(serviceAccountSecretVolumeName).MapKeys()) - volumes := make([]azaci.Volume, 0, len(*cgw.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes)) - for _, volume := range *cgw.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes { + volumes := make([]*azaciv2.Volume, 0, len(cgw.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Volumes)) + for _, volume := range cgw.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Volumes { if _, ok := serviceAccountSecretVolumeName[*volume.Name]; !ok { volumes = append(volumes, volume) } } - cgw.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes = &volumes + cgw.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Volumes = volumes } } -func getACIEnvVar(e v1.EnvVar) azaci.EnvironmentVariable { - var envVar azaci.EnvironmentVariable +func getACIEnvVar(e v1.EnvVar) *azaciv2.EnvironmentVariable { + var envVar azaciv2.EnvironmentVariable // If the variable is a secret, use SecureValue if e.ValueFrom != nil && e.ValueFrom.SecretKeyRef != nil { - envVar = azaci.EnvironmentVariable{ + envVar = azaciv2.EnvironmentVariable{ Name: &e.Name, SecureValue: &e.Value, } } else { - envVar = azaci.EnvironmentVariable{ + envVar = azaciv2.EnvironmentVariable{ Name: &e.Name, Value: &e.Value, } } - return envVar + return &envVar } diff --git a/pkg/provider/aci_init_container_test.go b/pkg/provider/aci_init_container_test.go index 13a93460..def043f1 100644 --- a/pkg/provider/aci_init_container_test.go +++ b/pkg/provider/aci_init_container_test.go @@ -26,17 +26,17 @@ func TestCreatePodWithInitContainers(t *testing.T) { aciMocks := createNewACIMock() aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error { - containers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers - initContainers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.InitContainers + containers := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers + initContainers := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.InitContainers assert.Check(t, cg != nil, "Container group is nil") assert.Check(t, containers != nil, "Containers should not be nil") assert.Check(t, initContainers != nil, "Container group is nil") assert.Check(t, is.Equal(len(containers), 2), "1 Container is expected") assert.Check(t, is.Equal(len(initContainers), 2), "2 init containers are expected") - assert.Check(t, initContainers[0].VolumeMounts != nil, "Volume mount should be present") - assert.Check(t, initContainers[0].EnvironmentVariables != nil, "Volume mount should be present") - assert.Check(t, initContainers[0].Command != nil, "Command mount should be present") - assert.Check(t, initContainers[0].Image != nil, "Image should be present") + assert.Check(t, initContainers[0].Properties.VolumeMounts != nil, "Volume mount should be present") + assert.Check(t, initContainers[0].Properties.EnvironmentVariables != nil, "Volume mount should be present") + assert.Check(t, initContainers[0].Properties.Command != nil, "Command mount should be present") + assert.Check(t, initContainers[0].Properties.Image != nil, "Image should be present") assert.Check(t, *initContainers[0].Name == initContainerName1, "Name should be correct") assert.Check(t, *initContainers[1].Name == initContainerName2, "Name should be correct") diff --git a/pkg/provider/aci_test.go b/pkg/provider/aci_test.go index 898f7b62..9a2bc65c 100644 --- a/pkg/provider/aci_test.go +++ b/pkg/provider/aci_test.go @@ -13,7 +13,7 @@ import ( "testing" "time" - azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/virtual-kubelet/azure-aci/pkg/auth" @@ -36,10 +36,11 @@ const ( ) var ( - gpuSKU = azaci.GpuSkuP100 + gpuSKU = azaciv2.GpuSKUP100 fakeRegion = getEnv("LOCATION", "westus2") creationTime = "2006-01-02 15:04:05.999999999 -0700 MST" azConfig auth.Config + runningState = "Running" ) func getEnv(key, fallback string) string { @@ -121,15 +122,15 @@ func TestCreatePodWithoutResourceSpec(t *testing.T) { aciMocks := createNewACIMock() aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error { - containers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers + containers := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers assert.Check(t, cg != nil, "Container group is nil") assert.Check(t, containers != nil, "Containers should not be nil") assert.Check(t, is.Equal(1, len(containers)), "1 Container is expected") assert.Check(t, is.Equal("nginx", *(containers[0]).Name), "Container nginx is expected") - assert.Check(t, (containers[0]).Resources.Requests != nil, "Container resource requests should not be nil") - assert.Check(t, is.Equal(1.0, *(containers[0]).Resources.Requests.CPU), "Request CPU is not expected") - assert.Check(t, is.Equal(1.5, *(containers[0]).Resources.Requests.MemoryInGB), "Request Memory is not expected") - assert.Check(t, is.Nil((containers[0]).Resources.Limits), "Limits should be nil") + assert.Check(t, containers[0].Properties.Resources.Requests != nil, "Container resource requests should not be nil") + assert.Check(t, is.Equal(1.0, *(containers[0]).Properties.Resources.Requests.CPU), "Request CPU is not expected") + assert.Check(t, is.Equal(1.5, *(containers[0]).Properties.Resources.Requests.MemoryInGB), "Request Memory is not expected") + assert.Check(t, is.Nil((containers[0]).Properties.Resources.Limits), "Limits should be nil") return nil } @@ -162,15 +163,15 @@ func TestCreatePodWithResourceRequestOnly(t *testing.T) { aciMocks := createNewACIMock() aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error { - containers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers + containers := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers assert.Check(t, cg != nil, "container group is nil") assert.Check(t, containers != nil, "container should not be nil") assert.Check(t, is.Equal(1, len(containers)), "only container is expected") assert.Check(t, is.Equal("nginx", *(containers[0]).Name), "Container nginx is expected") - assert.Check(t, containers[0].Resources.Requests != nil, "Container resource requests should not be nil") - assert.Check(t, is.Equal(1.98, *(containers[0]).Resources.Requests.CPU), "Request CPU is not expected") - assert.Check(t, is.Equal(3.4, *(containers[0]).Resources.Requests.MemoryInGB), "Request Memory is not expected") - assert.Check(t, is.Nil(containers[0].Resources.Limits), "Limits should be nil") + assert.Check(t, containers[0].Properties.Resources.Requests != nil, "Container resource requests should not be nil") + assert.Check(t, is.Equal(1.98, *(containers[0]).Properties.Resources.Requests.CPU), "Request CPU is not expected") + assert.Check(t, is.Equal(3.4, *(containers[0]).Properties.Resources.Requests.MemoryInGB), "Request Memory is not expected") + assert.Check(t, is.Nil(containers[0].Properties.Resources.Limits), "Limits should be nil") return nil } @@ -218,15 +219,15 @@ func TestCreatePodWithGPU(t *testing.T) { aciMocks := createNewACIMock() aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error { - containers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers + containers := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers assert.Check(t, containers != nil, "Containers should not be nil") assert.Check(t, is.Equal(1, len(containers)), "1 Container is expected") assert.Check(t, is.Equal("nginx", *(containers[0]).Name), "Container nginx is expected") - assert.Check(t, (containers[0]).Resources.Requests != nil, "Container resource requests should not be nil") - assert.Check(t, is.Equal(1.98, *(containers[0]).Resources.Requests.CPU), "Request CPU is not expected") - assert.Check(t, is.Equal(3.4, *(containers[0]).Resources.Requests.MemoryInGB), "Request Memory is not expected") - assert.Check(t, (containers[0]).Resources.Requests.Gpu != nil, "Requests GPU is not expected") - assert.Check(t, is.Equal(int32(10), *(containers[0]).Resources.Requests.Gpu.Count), "Requests GPU Count is not expected") + assert.Check(t, (containers[0]).Properties.Resources.Requests != nil, "Container resource requests should not be nil") + assert.Check(t, is.Equal(1.98, *(containers[0]).Properties.Resources.Requests.CPU), "Request CPU is not expected") + assert.Check(t, is.Equal(3.4, *(containers[0]).Properties.Resources.Requests.MemoryInGB), "Request Memory is not expected") + assert.Check(t, (containers[0]).Properties.Resources.Requests.Gpu != nil, "Requests GPU is not expected") + assert.Check(t, is.Equal(int32(10), *(containers[0]).Properties.Resources.Requests.Gpu.Count), "Requests GPU Count is not expected") return nil } @@ -272,18 +273,18 @@ func TestCreatePodWithGPUSKU(t *testing.T) { aciMocks := createNewACIMock() aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error { - containers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers + containers := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers assert.Check(t, cg != nil, "Container group is nil") assert.Check(t, containers != nil, "Containers should not be nil") assert.Check(t, is.Equal(1, len(containers)), "1 Container is expected") assert.Check(t, is.Equal("nginx", *(containers[0]).Name), "Container nginx is expected") - assert.Check(t, (containers[0]).Resources.Requests != nil, "Container resource requests should not be nil") - assert.Check(t, is.Equal(1.98, *(containers[0]).Resources.Requests.CPU), "Request CPU is not expected") - assert.Check(t, is.Equal(3.4, *(containers[0]).Resources.Requests.MemoryInGB), "Request Memory is not expected") - assert.Check(t, (containers[0]).Resources.Requests.Gpu != nil, "Requests GPU is not expected") - assert.Check(t, is.Equal(int32(1), *(containers[0]).Resources.Requests.Gpu.Count), "Requests GPU Count is not expected") - assert.Check(t, is.Equal(gpuSKU, (containers[0]).Resources.Requests.Gpu.Sku), "Requests GPU SKU is not expected") - assert.Check(t, (containers[0]).Resources.Limits.Gpu != nil, "Limits GPU is not expected") + assert.Check(t, (containers[0]).Properties.Resources.Requests != nil, "Container resource requests should not be nil") + assert.Check(t, is.Equal(1.98, *(containers[0]).Properties.Resources.Requests.CPU), "Request CPU is not expected") + assert.Check(t, is.Equal(3.4, *(containers[0]).Properties.Resources.Requests.MemoryInGB), "Request Memory is not expected") + assert.Check(t, (containers[0]).Properties.Resources.Requests.Gpu != nil, "Requests GPU is not expected") + assert.Check(t, is.Equal(int32(1), *(containers[0]).Properties.Resources.Requests.Gpu.Count), "Requests GPU Count is not expected") + assert.Check(t, is.Equal(gpuSKU, (containers[0]).Properties.Resources.Requests.Gpu.SKU), "Requests GPU SKU is not expected") + assert.Check(t, (containers[0]).Properties.Resources.Limits.Gpu != nil, "Limits GPU is not expected") return nil } @@ -332,16 +333,16 @@ func TestCreatePodWithResourceRequestAndLimit(t *testing.T) { aciMocks := createNewACIMock() aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error { - containers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers + containers := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers assert.Check(t, cg != nil, "Container group is nil") assert.Check(t, containers != nil, "Containers should not be nil") assert.Check(t, is.Equal(1, len(containers)), "1 Container is expected") assert.Check(t, is.Equal("nginx", *(containers[0]).Name), "Container nginx is expected") - assert.Check(t, (containers[0]).Resources.Requests != nil, "Container resource requests should not be nil") - assert.Check(t, is.Equal(0.99, *(containers[0]).Resources.Requests.CPU), "Request CPU is not expected") - assert.Check(t, is.Equal(1.5, *(containers[0]).Resources.Requests.MemoryInGB), "Request Memory is not expected") - assert.Check(t, is.Equal(3.999, *(containers[0]).Resources.Limits.CPU), "Limit CPU is not expected") - assert.Check(t, is.Equal(8.0, *(containers[0]).Resources.Limits.MemoryInGB), "Limit Memory is not expected") + assert.Check(t, (containers[0]).Properties.Resources.Requests != nil, "Container resource requests should not be nil") + assert.Check(t, is.Equal(0.99, *(containers[0]).Properties.Resources.Requests.CPU), "Request CPU is not expected") + assert.Check(t, is.Equal(1.5, *(containers[0]).Properties.Resources.Requests.MemoryInGB), "Request Memory is not expected") + assert.Check(t, is.Equal(3.999, *(containers[0]).Properties.Resources.Limits.CPU), "Limit CPU is not expected") + assert.Check(t, is.Equal(8.0, *(containers[0]).Properties.Resources.Limits.MemoryInGB), "Limit Memory is not expected") return nil } @@ -362,9 +363,9 @@ func TestCreatePodWithResourceRequestAndLimit(t *testing.T) { func TestGetPodsWithEmptyList(t *testing.T) { aciMocks := createNewACIMock() - aciMocks.MockGetContainerGroupList = func(ctx context.Context, resourceGroup string) (*[]azaci.ContainerGroup, error) { - var result []azaci.ContainerGroup - return &result, nil + aciMocks.MockGetContainerGroupList = func(ctx context.Context, resourceGroup string) ([]*azaciv2.ContainerGroup, error) { + var result []*azaciv2.ContainerGroup + return result, nil } provider, err := createTestProvider(aciMocks, nil) @@ -377,7 +378,6 @@ func TestGetPodsWithEmptyList(t *testing.T) { t.Fatal("Failed to get pods", err) } - assert.Check(t, pods != nil, "Response pods should not be nil") assert.Check(t, is.Equal(0, len(pods)), "No pod should be returned") } @@ -385,11 +385,12 @@ func TestGetPodsWithEmptyList(t *testing.T) { func TestGetPodsWithoutResourceRequestsLimits(t *testing.T) { aciMocks := createNewACIMock() - aciMocks.MockGetContainerGroupList = func(ctx context.Context, resourceGroup string) (*[]azaci.ContainerGroup, error) { + aciMocks.MockGetContainerGroupList = func(ctx context.Context, resourceGroup string) ([]*azaciv2.ContainerGroup, error) { cgName := "default-nginx" node := fakeNodeName provisioning := "Creating" - var cg = azaci.ContainerGroup{ + var cg = &azaciv2.ContainerGroup{ + ID: &cgName, Name: &cgName, Tags: map[string]*string{ "CreationTimestamp": &creationTime, @@ -399,14 +400,17 @@ func TestGetPodsWithoutResourceRequestsLimits(t *testing.T) { "NodeName": &node, "UID": &cgName, }, - ContainerGroupProperties: &azaci.ContainerGroupProperties{ + Properties: &azaciv2.ContainerGroupPropertiesProperties{ ProvisioningState: &provisioning, - Containers: testsutil.CreateACIContainersListObj("Running", "Initializing", testsutil.CgCreationTime.Add(time.Second*2), testsutil.CgCreationTime.Add(time.Second*3), true, false, false), + Containers: testsutil.CreateACIContainersListObj(runningState, "Initializing", testsutil.CgCreationTime.Add(time.Second*2), testsutil.CgCreationTime.Add(time.Second*3), true, false, false), + InstanceView: &azaciv2.ContainerGroupPropertiesInstanceView{ + State: &runningState, + }, }, } - var result []azaci.ContainerGroup + var result []*azaciv2.ContainerGroup result = append(result, cg) - return &result, nil + return result, nil } provider, err := createTestProvider(aciMocks, nil) @@ -440,18 +444,24 @@ func TestGetPodWithoutResourceRequestsLimits(t *testing.T) { aciMocks := createNewACIMock() aciMocks.MockGetContainerGroupInfo = - func(ctx context.Context, resourceGroup, namespace, name, nodeName string) (*azaci.ContainerGroup, error) { + func(ctx context.Context, resourceGroup, namespace, name, nodeName string) (*azaciv2.ContainerGroup, error) { return testsutil.CreateContainerGroupObj(podName, podNamespace, "Succeeded", - testsutil.CreateACIContainersListObj("Running", "Initializing", testsutil.CgCreationTime.Add(time.Second*2), testsutil.CgCreationTime.Add(time.Second*3), false, false, false), "Succeeded"), nil + testsutil.CreateACIContainersListObj(runningState, "Initializing", + testsutil.CgCreationTime.Add(time.Second*2), + testsutil.CgCreationTime.Add(time.Second*3), + false, false, false), "Succeeded"), nil } - aciMocks.MockGetContainerGroupList = func(ctx context.Context, resourceGroup string) (*[]azaci.ContainerGroup, error) { + aciMocks.MockGetContainerGroupList = func(ctx context.Context, resourceGroup string) ([]*azaciv2.ContainerGroup, error) { cg := testsutil.CreateContainerGroupObj(podName, podNamespace, "Succeeded", - testsutil.CreateACIContainersListObj("Running", "Initializing", testsutil.CgCreationTime.Add(time.Second*2), testsutil.CgCreationTime.Add(time.Second*3), false, false, false), "Succeeded") + testsutil.CreateACIContainersListObj(runningState, "Initializing", + testsutil.CgCreationTime.Add(time.Second*2), + testsutil.CgCreationTime.Add(time.Second*3), + false, false, false), "Succeeded") - var result []azaci.ContainerGroup - result = append(result, *cg) - return &result, nil + var result []*azaciv2.ContainerGroup + result = append(result, cg) + return result, nil } resourceManager, err := manager.NewResourceManager( @@ -475,8 +485,8 @@ func TestGetPodWithoutResourceRequestsLimits(t *testing.T) { t.Fatal("Failed to get pod", err) } - assert.Equal(t, ptrQuantity(resource.MustParse("0.99")).Value(), pod.Spec.Containers[0].Resources.Requests.Cpu().Value(), "Containers[0].Resources.Requests.CPU doesn't match") - assert.Equal(t, ptrQuantity(resource.MustParse("1.5G")).Value(), pod.Spec.Containers[0].Resources.Requests.Memory().Value(), "Containers[0].Resources.Requests.Memory doesn't match") + assert.Equal(t, ptrQuantity(resource.MustParse("0.99")).Value(), pod.Spec.Containers[0].Resources.Requests.Cpu().Value(), "Containers[0].Properties.Resources.Requests.CPU doesn't match") + assert.Equal(t, ptrQuantity(resource.MustParse("1.5G")).Value(), pod.Spec.Containers[0].Resources.Requests.Memory().Value(), "Containers[0].Properties.Resources.Requests.Memory doesn't match") } func TestPodToACISecretEnvVar(t *testing.T) { @@ -540,15 +550,15 @@ func setAuthConfig() error { } func createNewACIMock() *MockACIProvider { - return NewMockACIProvider(func(ctx context.Context, region string) (*[]azaci.Capabilities, error) { + return NewMockACIProvider(func(ctx context.Context, region string) ([]*azaciv2.Capabilities, error) { gpu := "P100" - capability := azaci.Capabilities{ + capability := &azaciv2.Capabilities{ Location: ®ion, Gpu: &gpu, } - var result []azaci.Capabilities + var result []*azaciv2.Capabilities result = append(result, capability) - return &result, nil + return result, nil }) } @@ -635,16 +645,16 @@ func TestCreatePodWithNamedLivenessProbe(t *testing.T) { aciMocks := createNewACIMock() aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error { - containers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers - - assert.Check(t, (containers)[0].LivenessProbe != nil, "Liveness probe expected") - assert.Check(t, is.Equal(int32(10), *(containers)[0].LivenessProbe.InitialDelaySeconds), "Initial Probe Delay doesn't match") - assert.Check(t, is.Equal(int32(5), *(containers)[0].LivenessProbe.PeriodSeconds), "Probe Period doesn't match") - assert.Check(t, is.Equal(int32(60), *(containers)[0].LivenessProbe.TimeoutSeconds), "Probe Timeout doesn't match") - assert.Check(t, is.Equal(int32(3), *(containers)[0].LivenessProbe.SuccessThreshold), "Probe Success Threshold doesn't match") - assert.Check(t, is.Equal(int32(5), *(containers)[0].LivenessProbe.FailureThreshold), "Probe Failure Threshold doesn't match") - assert.Check(t, (*cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers)[0].LivenessProbe.HTTPGet != nil, "Expected an HTTP Get Probe") - assert.Check(t, is.Equal(int32(8080), *(containers)[0].LivenessProbe.HTTPGet.Port), "Expected Port to be 8080") + containers := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers + + assert.Check(t, (containers)[0].Properties.LivenessProbe != nil, "Liveness probe expected") + assert.Check(t, is.Equal(int32(10), *(containers)[0].Properties.LivenessProbe.InitialDelaySeconds), "Initial Probe Delay doesn't match") + assert.Check(t, is.Equal(int32(5), *(containers)[0].Properties.LivenessProbe.PeriodSeconds), "Probe Period doesn't match") + assert.Check(t, is.Equal(int32(60), *(containers)[0].Properties.LivenessProbe.TimeoutSeconds), "Probe Timeout doesn't match") + assert.Check(t, is.Equal(int32(3), *(containers)[0].Properties.LivenessProbe.SuccessThreshold), "Probe Success Threshold doesn't match") + assert.Check(t, is.Equal(int32(5), *(containers)[0].Properties.LivenessProbe.FailureThreshold), "Probe Failure Threshold doesn't match") + assert.Check(t, (cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers)[0].Properties.LivenessProbe.HTTPGet != nil, "Expected an HTTP Get Probe") + assert.Check(t, is.Equal(int32(8080), *(containers)[0].Properties.LivenessProbe.HTTPGet.Port), "Expected Port to be 8080") return nil } @@ -666,18 +676,18 @@ func TestCreatePodWithLivenessProbe(t *testing.T) { aciMocks := createNewACIMock() aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error { - containers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers + containers := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers assert.Check(t, cg != nil, "Container group is nil") assert.Check(t, containers != nil, "Containers should not be nil") assert.Check(t, is.Equal(1, len(containers)), "1 Container is expected") assert.Check(t, is.Equal("nginx", *(containers[0]).Name), "Container nginx is expected") - assert.Check(t, (containers)[0].LivenessProbe != nil, "Liveness probe expected") - assert.Check(t, is.Equal(int32(10), *(containers)[0].LivenessProbe.InitialDelaySeconds), "Initial Probe Delay doesn't match") - assert.Check(t, is.Equal(int32(5), *(containers)[0].LivenessProbe.PeriodSeconds), "Probe Period doesn't match") - assert.Check(t, is.Equal(int32(60), *(containers)[0].LivenessProbe.TimeoutSeconds), "Probe Timeout doesn't match") - assert.Check(t, is.Equal(int32(3), *(containers)[0].LivenessProbe.SuccessThreshold), "Probe Success Threshold doesn't match") - assert.Check(t, is.Equal(int32(5), *(containers)[0].LivenessProbe.FailureThreshold), "Probe Failure Threshold doesn't match") - assert.Check(t, (containers)[0].LivenessProbe.HTTPGet != nil, "Expected an HTTP Get Probe") + assert.Check(t, (containers)[0].Properties.LivenessProbe != nil, "Liveness probe expected") + assert.Check(t, is.Equal(int32(10), *(containers)[0].Properties.LivenessProbe.InitialDelaySeconds), "Initial Probe Delay doesn't match") + assert.Check(t, is.Equal(int32(5), *(containers)[0].Properties.LivenessProbe.PeriodSeconds), "Probe Period doesn't match") + assert.Check(t, is.Equal(int32(60), *(containers)[0].Properties.LivenessProbe.TimeoutSeconds), "Probe Timeout doesn't match") + assert.Check(t, is.Equal(int32(3), *(containers)[0].Properties.LivenessProbe.SuccessThreshold), "Probe Success Threshold doesn't match") + assert.Check(t, is.Equal(int32(5), *(containers)[0].Properties.LivenessProbe.FailureThreshold), "Probe Failure Threshold doesn't match") + assert.Check(t, (containers)[0].Properties.LivenessProbe.HTTPGet != nil, "Expected an HTTP Get Probe") return nil } @@ -694,6 +704,85 @@ func TestCreatePodWithLivenessProbe(t *testing.T) { } } +func TestGetProbe(t *testing.T) { + cases := []struct { + description string + podProbe *v1.Probe + podPorts []v1.ContainerPort + expectedCGProbe *azaciv2.ContainerProbe + expectedError error + }{ + { + description: "has_no_probe", + podProbe: testsutil.CreatePodProbeObj(false, false), + podPorts: nil, + expectedCGProbe: nil, + expectedError: fmt.Errorf("probe must specify one of \"exec\" and \"httpGet\""), + }, { + description: "has_httpGet_and_exec", + podProbe: testsutil.CreatePodProbeObj(true, true), + podPorts: nil, + expectedCGProbe: nil, + expectedError: fmt.Errorf("probe may not specify more than one of \"exec\" and \"httpGet\""), + }, { + description: "has_httpGet_wrong_port_info", + podProbe: testsutil.CreatePodProbeObj(true, false), + podPorts: testsutil.CreateContainerPortObj("https", 8888), + expectedCGProbe: nil, + expectedError: fmt.Errorf("unable to find named port: %s", "http"), + }, { + description: "has_exec_with_port_info", + podProbe: testsutil.CreatePodProbeObj(false, true), + podPorts: testsutil.CreateContainerPortObj("http", 8080), + expectedCGProbe: testsutil.CreateCGProbeObj(false, true), + expectedError: nil, + }, + { + description: "has_exec_without_port_info", + podProbe: testsutil.CreatePodProbeObj(false, true), + podPorts: nil, + expectedCGProbe: testsutil.CreateCGProbeObj(false, true), + expectedError: nil, + }, + { + description: "has_httpGet_with_port_info", + podProbe: testsutil.CreatePodProbeObj(true, false), + podPorts: testsutil.CreateContainerPortObj("http", 8080), + expectedCGProbe: testsutil.CreateCGProbeObj(true, false), + expectedError: nil, + }, + { + description: "has_httpGet_without_port_info", + podProbe: testsutil.CreatePodProbeObj(true, false), + podPorts: nil, + expectedCGProbe: nil, + expectedError: fmt.Errorf("unable to find named port: %s", "http"), + }, + { + description: "has_httpGet_with_wrong_port_info", + podProbe: testsutil.CreatePodProbeObj(true, false), + podPorts: testsutil.CreateContainerPortObj("https", 8080), + expectedCGProbe: nil, + expectedError: fmt.Errorf("unable to find named port: %s", "http"), + }, + } + for _, tc := range cases { + t.Run(tc.description, func(t *testing.T) { + + cgProbe, err := getProbe(tc.podProbe, tc.podPorts) + + if tc.expectedCGProbe != nil { + assert.DeepEqual(t, tc.expectedCGProbe, cgProbe) + } + if tc.expectedError == nil { + assert.NilError(t, tc.expectedError, err) + } else { + assert.Equal(t, tc.expectedError.Error(), err.Error()) + } + }) + } +} + func TestCreatePodWithReadinessProbe(t *testing.T) { podName := "pod-" + uuid.New().String() podNamespace := "ns-" + uuid.New().String() @@ -701,18 +790,18 @@ func TestCreatePodWithReadinessProbe(t *testing.T) { aciMocks := createNewACIMock() aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error { - containers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers + containers := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers assert.Check(t, cg != nil, "Container group is nil") assert.Check(t, containers != nil, "Containers should not be nil") assert.Check(t, is.Equal(1, len(containers)), "1 Container is expected") assert.Check(t, is.Equal("nginx", *(containers[0]).Name), "Container nginx is expected") - assert.Check(t, (containers)[0].ReadinessProbe != nil, "Readiness probe expected") - assert.Check(t, is.Equal(int32(10), *(containers)[0].ReadinessProbe.InitialDelaySeconds), "Initial Probe Delay doesn't match") - assert.Check(t, is.Equal(int32(5), *(containers)[0].ReadinessProbe.PeriodSeconds), "Probe Period doesn't match") - assert.Check(t, is.Equal(int32(60), *(containers)[0].ReadinessProbe.TimeoutSeconds), "Probe Timeout doesn't match") - assert.Check(t, is.Equal(int32(3), *(containers)[0].ReadinessProbe.SuccessThreshold), "Probe Success Threshold doesn't match") - assert.Check(t, is.Equal(int32(5), *(containers)[0].ReadinessProbe.FailureThreshold), "Probe Failure Threshold doesn't match") - assert.Check(t, (containers)[0].ReadinessProbe.HTTPGet != nil, "Expected an HTTP Get Probe") + assert.Check(t, (containers)[0].Properties.ReadinessProbe != nil, "Readiness probe expected") + assert.Check(t, is.Equal(int32(10), *(containers)[0].Properties.ReadinessProbe.InitialDelaySeconds), "Initial Probe Delay doesn't match") + assert.Check(t, is.Equal(int32(5), *(containers)[0].Properties.ReadinessProbe.PeriodSeconds), "Probe Period doesn't match") + assert.Check(t, is.Equal(int32(60), *(containers)[0].Properties.ReadinessProbe.TimeoutSeconds), "Probe Timeout doesn't match") + assert.Check(t, is.Equal(int32(3), *(containers)[0].Properties.ReadinessProbe.SuccessThreshold), "Probe Success Threshold doesn't match") + assert.Check(t, is.Equal(int32(5), *(containers)[0].Properties.ReadinessProbe.FailureThreshold), "Probe Failure Threshold doesn't match") + assert.Check(t, (containers)[0].Properties.ReadinessProbe.HTTPGet != nil, "Expected an HTTP Get Probe") return nil } @@ -835,9 +924,9 @@ func TestCreatedPodWithContainerPort(t *testing.T) { aciMocks := createNewACIMock() aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error { - containers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers - container1Ports := *(containers)[0].Ports - container2Ports := *(containers)[1].Ports + containers := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers + container1Ports := containers[0].Properties.Ports + container2Ports := containers[1].Properties.Ports assert.Check(t, cg != nil, "Container group is nil") assert.Check(t, containers != nil, "Containers should not be nil") assert.Check(t, is.Equal(2, len(containers)), "2 Containers is expected") @@ -895,7 +984,7 @@ func TestGetPodWithContainerID(t *testing.T) { aciMocks := createNewACIMock() cgID := "" - aciMocks.MockGetContainerGroupInfo = func(ctx context.Context, resourceGroup, namespace, name, nodeName string) (*azaci.ContainerGroup, error) { + aciMocks.MockGetContainerGroupInfo = func(ctx context.Context, resourceGroup, namespace, name, nodeName string) (*azaciv2.ContainerGroup, error) { cg := testsutil.CreateContainerGroupObj(podName, podNamespace, "Succeeded", testsutil.CreateACIContainersListObj("Running", "Initializing", testsutil.CgCreationTime.Add(time.Second*2), testsutil.CgCreationTime.Add(time.Second*3), false, false, false), "Succeeded") diff --git a/pkg/provider/aci_volumes.go b/pkg/provider/aci_volumes.go index 5149a77a..b947b9b1 100644 --- a/pkg/provider/aci_volumes.go +++ b/pkg/provider/aci_volumes.go @@ -9,13 +9,13 @@ import ( "encoding/base64" "fmt" - azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" "github.com/virtual-kubelet/virtual-kubelet/log" v1 "k8s.io/api/core/v1" k8serr "k8s.io/apimachinery/pkg/api/errors" ) -func (p *ACIProvider) getAzureFileCSI(volume v1.Volume, namespace string) (*azaci.Volume, error) { +func (p *ACIProvider) getAzureFileCSI(volume v1.Volume, namespace string) (*azaciv2.Volume, error) { var secretName, shareName string if volume.CSI.VolumeAttributes != nil && len(volume.CSI.VolumeAttributes) != 0 { for k, v := range volume.CSI.VolumeAttributes { @@ -47,17 +47,17 @@ func (p *ACIProvider) getAzureFileCSI(volume v1.Volume, namespace string) (*azac storageAccountNameStr := string(secret.Data[azureFileStorageAccountName]) storageAccountKeyStr := string(secret.Data[azureFileStorageAccountKey]) - return &azaci.Volume{ + return &azaciv2.Volume{ Name: &volume.Name, - AzureFile: &azaci.AzureFileVolume{ + AzureFile: &azaciv2.AzureFileVolume{ ShareName: &shareName, StorageAccountName: &storageAccountNameStr, StorageAccountKey: &storageAccountKeyStr, }}, nil } -func (p *ACIProvider) getVolumes(ctx context.Context, pod *v1.Pod) ([]azaci.Volume, error) { - volumes := make([]azaci.Volume, 0, len(pod.Spec.Volumes)) +func (p *ACIProvider) getVolumes(ctx context.Context, pod *v1.Pod) ([]*azaciv2.Volume, error) { + volumes := make([]*azaciv2.Volume, 0, len(pod.Spec.Volumes)) podVolumes := pod.Spec.Volumes for i := range podVolumes { // Handle the case for Azure File CSI driver @@ -68,7 +68,7 @@ func (p *ACIProvider) getVolumes(ctx context.Context, pod *v1.Pod) ([]azaci.Volu if err != nil { return nil, err } - volumes = append(volumes, *csiVolume) + volumes = append(volumes, csiVolume) continue } else { return nil, fmt.Errorf("pod %s requires volume %s which is of an unsupported type %s", pod.Name, podVolumes[i].Name, podVolumes[i].CSI.Driver) @@ -88,9 +88,9 @@ func (p *ACIProvider) getVolumes(ctx context.Context, pod *v1.Pod) ([]azaci.Volu storageAccountNameStr := string(secret.Data[azureFileStorageAccountName]) storageAccountKeyStr := string(secret.Data[azureFileStorageAccountKey]) - volumes = append(volumes, azaci.Volume{ + volumes = append(volumes, &azaciv2.Volume{ Name: &podVolumes[i].Name, - AzureFile: &azaci.AzureFileVolume{ + AzureFile: &azaciv2.AzureFileVolume{ ShareName: &podVolumes[i].AzureFile.ShareName, ReadOnly: &podVolumes[i].AzureFile.ReadOnly, StorageAccountName: &storageAccountNameStr, @@ -103,7 +103,7 @@ func (p *ACIProvider) getVolumes(ctx context.Context, pod *v1.Pod) ([]azaci.Volu // Handle the case for the EmptyDir. if podVolumes[i].EmptyDir != nil { log.G(ctx).Info("empty volume name ", podVolumes[i].Name) - volumes = append(volumes, azaci.Volume{ + volumes = append(volumes, &azaciv2.Volume{ Name: &podVolumes[i].Name, EmptyDir: map[string]interface{}{}, }) @@ -112,9 +112,9 @@ func (p *ACIProvider) getVolumes(ctx context.Context, pod *v1.Pod) ([]azaci.Volu // Handle the case for GitRepo volume. if podVolumes[i].GitRepo != nil { - volumes = append(volumes, azaci.Volume{ + volumes = append(volumes, &azaciv2.Volume{ Name: &podVolumes[i].Name, - GitRepo: &azaci.GitRepoVolume{ + GitRepo: &azaciv2.GitRepoVolume{ Directory: &podVolumes[i].GitRepo.Directory, Repository: &podVolumes[i].GitRepo.Repository, Revision: &podVolumes[i].GitRepo.Revision, @@ -128,7 +128,7 @@ func (p *ACIProvider) getVolumes(ctx context.Context, pod *v1.Pod) ([]azaci.Volu paths := make(map[string]*string) secret, err := p.resourceManager.GetSecret(podVolumes[i].Secret.SecretName, pod.Namespace) if podVolumes[i].Secret.Optional != nil && !*podVolumes[i].Secret.Optional && k8serr.IsNotFound(err) { - return nil, fmt.Errorf("Secret %s is required by Pod %s and does not exist", podVolumes[i].Secret.SecretName, pod.Name) + return nil, fmt.Errorf("secret %s is required by Pod %s and does not exist", podVolumes[i].Secret.SecretName, pod.Name) } if secret == nil { continue @@ -140,7 +140,7 @@ func (p *ACIProvider) getVolumes(ctx context.Context, pod *v1.Pod) ([]azaci.Volu } if len(paths) != 0 { - volumes = append(volumes, azaci.Volume{ + volumes = append(volumes, &azaciv2.Volume{ Name: &podVolumes[i].Name, Secret: paths, }) @@ -169,7 +169,7 @@ func (p *ACIProvider) getVolumes(ctx context.Context, pod *v1.Pod) ([]azaci.Volu } if len(paths) != 0 { - volumes = append(volumes, azaci.Volume{ + volumes = append(volumes, &azaciv2.Volume{ Name: &podVolumes[i].Name, Secret: paths, }) @@ -271,7 +271,7 @@ func (p *ACIProvider) getVolumes(ctx context.Context, pod *v1.Pod) ([]azaci.Volu } } if len(paths) != 0 { - volumes = append(volumes, azaci.Volume{ + volumes = append(volumes, &azaciv2.Volume{ Name: &podVolumes[i].Name, Secret: paths, }) diff --git a/pkg/provider/aci_volumes_test.go b/pkg/provider/aci_volumes_test.go index 1bcfc896..6743890b 100644 --- a/pkg/provider/aci_volumes_test.go +++ b/pkg/provider/aci_volumes_test.go @@ -10,7 +10,7 @@ import ( "fmt" "testing" - azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/virtual-kubelet/azure-aci/pkg/client" @@ -61,14 +61,14 @@ func TestCreatedPodWithAzureFilesVolume(t *testing.T) { initEnabled := provider.enabledFeatures.IsEnabled(context.TODO(), featureflag.InitContainerFeature) aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error { - containers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers + containers := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers // Check only if init container feature is enabled if initEnabled { - initContainers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.InitContainers - assert.Check(t, initContainers[0].VolumeMounts != nil, "Volume mount should be present") - assert.Check(t, initContainers[0].EnvironmentVariables != nil, "Volume mount should be present") - assert.Check(t, initContainers[0].Command != nil, "Command mount should be present") - assert.Check(t, initContainers[0].Image != nil, "Image should be present") + initContainers := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.InitContainers + assert.Check(t, initContainers[0].Properties.VolumeMounts != nil, "Volume mount should be present") + assert.Check(t, initContainers[0].Properties.EnvironmentVariables != nil, "Volume mount should be present") + assert.Check(t, initContainers[0].Properties.Command != nil, "Command mount should be present") + assert.Check(t, initContainers[0].Properties.Image != nil, "Image should be present") assert.Check(t, *initContainers[0].Name == initContainerName, "Name should be correct") } @@ -76,11 +76,11 @@ func TestCreatedPodWithAzureFilesVolume(t *testing.T) { assert.Check(t, containers != nil, "Containers should not be nil") assert.Check(t, is.Equal(1, len(containers)), "1 Container is expected") assert.Check(t, is.Equal("nginx", *(containers)[0].Name), "Container nginx is expected") - assert.Check(t, is.Equal(3, len(*cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes)), "volume count not match") - assert.Check(t, is.Equal(azureFileVolumeName1, *(*cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes)[1].Name), "volume name is not matched") - assert.Check(t, is.Equal(fakeShareName1, *(*cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes)[1].AzureFile.ShareName), "volume share name is not matched") - assert.Check(t, is.Equal(azureFileVolumeName2, *(*cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes)[2].Name), "volume name is not matched") - assert.Check(t, is.Equal(fakeShareName2, *(*cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes)[2].AzureFile.ShareName), "volume share name is not matched") + assert.Check(t, is.Equal(3, len(cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Volumes)), "volume count not match") + assert.Check(t, is.Equal(azureFileVolumeName1, *(cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Volumes)[1].Name), "volume name is not matched") + assert.Check(t, is.Equal(fakeShareName1, *(cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Volumes)[1].AzureFile.ShareName), "volume share name is not matched") + assert.Check(t, is.Equal(azureFileVolumeName2, *(cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Volumes)[2].Name), "volume name is not matched") + assert.Check(t, is.Equal(fakeShareName2, *(cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Volumes)[2].AzureFile.ShareName), "volume share name is not matched") return nil } @@ -250,9 +250,9 @@ func TestCreatePodWithProjectedVolume(t *testing.T) { encodedSecretVal := base64.StdEncoding.EncodeToString([]byte("fake-ca-data")) aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error { - containers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers - volumes := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes - certVal := (*cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes)[2].Secret["ca.crt"] + containers := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers + volumes := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Volumes + certVal := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Volumes[2].Secret["ca.crt"] assert.Check(t, cg != nil, "Container group is nil") assert.Check(t, containers != nil, "Containers should not be nil") assert.Check(t, is.Equal(1, len(containers)), "1 Container is expected") @@ -264,29 +264,27 @@ func TestCreatePodWithProjectedVolume(t *testing.T) { return nil } - aciMocks.MockGetContainerGroupInfo = func(ctx context.Context, resourceGroup, namespace, name, nodeName string) (*azaci.ContainerGroup, error) { + aciMocks.MockGetContainerGroupInfo = func(ctx context.Context, resourceGroup, namespace, name, nodeName string) (*azaciv2.ContainerGroup, error) { caStr := "ca.crt" - node := fakeNodeName - cgName := "nginx" provisioningState := "Creating" - return &azaci.ContainerGroup{ + return &azaciv2.ContainerGroup{ Tags: map[string]*string{ "CreationTimestamp": &creationTime, "PodName": &podName, "Namespace": &podNamespace, - "ClusterName": &node, - "NodeName": &node, + "ClusterName": &nodeName, + "NodeName": &nodeName, "UID": &podName, }, Name: &cgName, - ContainerGroupProperties: &azaci.ContainerGroupProperties{ + Properties: &azaciv2.ContainerGroupPropertiesProperties{ ProvisioningState: &provisioningState, - Volumes: &[]azaci.Volume{ + Volumes: []*azaciv2.Volume{ { Name: &emptyVolumeName, }, { Name: &azureFileVolumeName, - AzureFile: &azaci.AzureFileVolume{ + AzureFile: &azaciv2.AzureFileVolume{ ShareName: &fakeShareName1, }, }, { @@ -343,12 +341,12 @@ func TestCreatePodWithCSIVolume(t *testing.T) { aciMocks := createNewACIMock() aciMocks.MockCreateContainerGroup = func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error { - containers := *cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Containers + containers := cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Containers assert.Check(t, cg != nil, "Container group is nil") assert.Check(t, containers != nil, "Containers should not be nil") assert.Check(t, is.Equal(1, len(containers)), "1 Container is expected") assert.Check(t, is.Equal("nginx", *(containers[0]).Name), "Container nginx is expected") - assert.Check(t, is.Equal(2, len(*cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Volumes)), "volume count not match") + assert.Check(t, is.Equal(2, len(cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.Properties.Volumes)), "volume count not match") return nil } diff --git a/pkg/provider/containergroup_to_pod.go b/pkg/provider/containergroup_to_pod.go index 1f555920..58ad46a2 100644 --- a/pkg/provider/containergroup_to_pod.go +++ b/pkg/provider/containergroup_to_pod.go @@ -5,18 +5,20 @@ Licensed under the Apache 2.0 license. package provider import ( + "context" "time" - azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" "github.com/pkg/errors" "github.com/virtual-kubelet/azure-aci/pkg/tests" "github.com/virtual-kubelet/azure-aci/pkg/util" "github.com/virtual-kubelet/azure-aci/pkg/validation" + "github.com/virtual-kubelet/virtual-kubelet/log" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func (p *ACIProvider) containerGroupToPod(cg *azaci.ContainerGroup) (*v1.Pod, error) { +func (p *ACIProvider) containerGroupToPod(ctx context.Context, cg *azaciv2.ContainerGroup) (*v1.Pod, error) { //cg is validated pod, err := p.resourceManager.GetPod(*cg.Name, *cg.Tags["Namespace"]) if err != nil { @@ -25,7 +27,7 @@ func (p *ACIProvider) containerGroupToPod(cg *azaci.ContainerGroup) (*v1.Pod, er updatedPod := pod.DeepCopy() - podState, err := p.getPodStatusFromContainerGroup(cg) + podState, err := p.getPodStatusFromContainerGroup(ctx, cg) if err != nil { return nil, err } @@ -35,41 +37,48 @@ func (p *ACIProvider) containerGroupToPod(cg *azaci.ContainerGroup) (*v1.Pod, er return updatedPod, nil } -func (p *ACIProvider) getPodStatusFromContainerGroup(cg *azaci.ContainerGroup) (*v1.PodStatus, error) { +func (p *ACIProvider) getPodStatusFromContainerGroup(ctx context.Context, cg *azaciv2.ContainerGroup) (*v1.PodStatus, error) { + logger := log.G(ctx).WithField("method", "getPodStatusFromContainerGroup") + // cg is validated allReady := true - var firstContainerStartTime, lastUpdateTime metav1.Time + var firstContainerStartTime, lastUpdateTime time.Time - containerStatuses := make([]v1.ContainerStatus, 0, len(*cg.Containers)) - containersList := *cg.Containers + containerStatuses := make([]v1.ContainerStatus, 0, len(cg.Properties.Containers)) + containersList := cg.Properties.Containers for i := range containersList { - err := validation.ValidateContainer(containersList[i]) + err := validation.ValidateContainer(ctx, containersList[i]) + logger.Debugf("container %s has missing fields. retrying the validation...", *containersList[i].Name) if err != nil { return nil, err } - firstContainerStartTime := metav1.NewTime(containersList[0].InstanceView.CurrentState.StartTime.Time) - lastUpdateTime := firstContainerStartTime + + // init the firstContainerStartTime & lastUpdateTime + if i == 0 { + firstContainerStartTime = *containersList[0].Properties.InstanceView.CurrentState.StartTime + lastUpdateTime = firstContainerStartTime + } containerStatus := v1.ContainerStatus{ Name: *containersList[i].Name, - State: aciContainerStateToContainerState(containersList[i].InstanceView.CurrentState), - LastTerminationState: aciContainerStateToContainerState(containersList[i].InstanceView.PreviousState), - Ready: getPodPhaseFromACIState(*containersList[i].InstanceView.CurrentState.State) == v1.PodRunning, - RestartCount: *containersList[i].InstanceView.RestartCount, - Image: *containersList[i].Image, + State: aciContainerStateToContainerState(containersList[i].Properties.InstanceView.CurrentState), + LastTerminationState: aciContainerStateToContainerState(containersList[i].Properties.InstanceView.PreviousState), + Ready: getPodPhaseFromACIState(*containersList[i].Properties.InstanceView.CurrentState.State) == v1.PodRunning, + RestartCount: *containersList[i].Properties.InstanceView.RestartCount, + Image: *containersList[i].Properties.Image, ImageID: "", ContainerID: util.GetContainerID(cg.ID, containersList[i].Name), } - if getPodPhaseFromACIState(*containersList[i].InstanceView.CurrentState.State) != v1.PodRunning && - getPodPhaseFromACIState(*containersList[i].InstanceView.CurrentState.State) != v1.PodSucceeded { + if getPodPhaseFromACIState(*containersList[i].Properties.InstanceView.CurrentState.State) != v1.PodRunning && + getPodPhaseFromACIState(*containersList[i].Properties.InstanceView.CurrentState.State) != v1.PodSucceeded { allReady = false } - containerStartTime := metav1.NewTime(containersList[i].ContainerProperties.InstanceView.CurrentState.StartTime.Time) - if containerStartTime.Time.After(lastUpdateTime.Time) { - lastUpdateTime = containerStartTime + containerStartTime := containersList[i].Properties.InstanceView.CurrentState.StartTime + if containerStartTime.After(lastUpdateTime) { + lastUpdateTime = *containerStartTime } // Add to containerStatuses @@ -82,8 +91,9 @@ func (p *ACIProvider) getPodStatusFromContainerGroup(cg *azaci.ContainerGroup) ( } podIp := "" - if cg.OsType != azaci.OperatingSystemTypesWindows { - podIp = *cg.IPAddress.IP + if cg.Properties.OSType != nil && + *cg.Properties.OSType != azaciv2.OperatingSystemTypesWindows { + podIp = *cg.Properties.IPAddress.IP } return &v1.PodStatus{ Phase: getPodPhaseFromACIState(*aciState), @@ -92,27 +102,35 @@ func (p *ACIProvider) getPodStatusFromContainerGroup(cg *azaci.ContainerGroup) ( Reason: "", HostIP: p.internalIP, PodIP: podIp, - StartTime: &firstContainerStartTime, + StartTime: &metav1.Time{Time: firstContainerStartTime}, ContainerStatuses: containerStatuses, }, nil } -func aciContainerStateToContainerState(cs *azaci.ContainerState) v1.ContainerState { +func aciContainerStateToContainerState(cs *azaciv2.ContainerState) v1.ContainerState { // cg container state is validated + finishTime := time.Time{} + startTime := time.Time{} + if cs.StartTime != nil { + startTime = *cs.StartTime + } + if cs.FinishTime != nil { + finishTime = *cs.FinishTime + } switch *cs.State { case "Running": return v1.ContainerState{ Running: &v1.ContainerStateRunning{ - StartedAt: metav1.NewTime(cs.StartTime.Time), + StartedAt: metav1.NewTime(startTime), }, } // Handle the case of completion. case "Succeeded": return v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ - StartedAt: metav1.NewTime(cs.StartTime.Time), + StartedAt: metav1.NewTime(startTime), Reason: "Completed", - FinishedAt: metav1.NewTime(cs.FinishTime.Time), + FinishedAt: metav1.NewTime(finishTime), }, } // Handle the case where the container failed. @@ -122,8 +140,8 @@ func aciContainerStateToContainerState(cs *azaci.ContainerState) v1.ContainerSta ExitCode: *cs.ExitCode, Reason: *cs.State, Message: *cs.DetailStatus, - StartedAt: metav1.NewTime(cs.StartTime.Time), - FinishedAt: metav1.NewTime(cs.FinishTime.Time), + StartedAt: metav1.NewTime(startTime), + FinishedAt: metav1.NewTime(finishTime), }, } // Handle windows container with no prev state @@ -170,7 +188,7 @@ func getPodPhaseFromACIState(state string) v1.PodPhase { return v1.PodUnknown } -func getPodConditionsFromACIState(state string, creationTime, lastUpdateTime metav1.Time, allReady bool) []v1.PodCondition { +func getPodConditionsFromACIState(state string, creationTime, lastUpdateTime time.Time, allReady bool) []v1.PodCondition { // cg state is validated switch state { case "Running", "Succeeded": @@ -185,32 +203,32 @@ func getPodConditionsFromACIState(state string, creationTime, lastUpdateTime met { Type: v1.PodReady, Status: readyConditionStatus, - LastTransitionTime: readyConditionTime, + LastTransitionTime: metav1.Time{Time: readyConditionTime}, }, { Type: v1.PodInitialized, Status: v1.ConditionTrue, - LastTransitionTime: creationTime, + LastTransitionTime: metav1.Time{Time: creationTime}, }, { Type: v1.PodScheduled, Status: v1.ConditionTrue, - LastTransitionTime: creationTime, + LastTransitionTime: metav1.Time{Time: creationTime}, }, } } return []v1.PodCondition{} } -func getACIResourceMetaFromContainerGroup(cg *azaci.ContainerGroup) (*string, metav1.Time, error) { +func getACIResourceMetaFromContainerGroup(cg *azaciv2.ContainerGroup) (*string, time.Time, error) { // cg is validated // Use the Provisioning State if it's not Succeeded, // otherwise use the state of the instance. - aciState := cg.ContainerGroupProperties.ProvisioningState + aciState := cg.Properties.ProvisioningState if *aciState == "Succeeded" { - aciState = cg.ContainerGroupProperties.InstanceView.State + aciState = cg.Properties.InstanceView.State } - var creationTime metav1.Time + var creationTime time.Time // cg tags is validated ts := *cg.Tags["CreationTimestamp"] @@ -218,9 +236,9 @@ func getACIResourceMetaFromContainerGroup(cg *azaci.ContainerGroup) (*string, me if ts != "" { t, err := time.Parse(tests.TimeLayout, ts) if err != nil { - return nil, metav1.Now(), errors.Errorf("unable to parse the creation timestamp for container group %s", *cg.Name) + return nil, time.Now(), errors.Errorf("unable to parse the creation timestamp for container group %s", *cg.Name) } - creationTime = metav1.NewTime(t) + creationTime = t } return aciState, creationTime, nil diff --git a/pkg/provider/containergroup_to_pod_test.go b/pkg/provider/containergroup_to_pod_test.go index 1f72db05..766db3a8 100644 --- a/pkg/provider/containergroup_to_pod_test.go +++ b/pkg/provider/containergroup_to_pod_test.go @@ -5,10 +5,11 @@ Licensed under the Apache 2.0 license. package provider import ( + "context" "testing" "time" - azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" testutil "github.com/virtual-kubelet/azure-aci/pkg/tests" "gotest.tools/assert" v1 "k8s.io/api/core/v1" @@ -30,7 +31,7 @@ func TestContainerGroupToPodStatus(t *testing.T) { } cases := []struct { description string - containerGroup *azaci.ContainerGroup + containerGroup *azaciv2.ContainerGroup expectedPodPhase v1.PodPhase expectedPodConditions []v1.PodCondition }{ @@ -49,7 +50,7 @@ func TestContainerGroupToPodStatus(t *testing.T) { } for _, tc := range cases { t.Run(tc.description, func(t *testing.T) { - expectedStatus, err := provider.getPodStatusFromContainerGroup(tc.containerGroup) + expectedStatus, err := provider.getPodStatusFromContainerGroup(context.TODO(), tc.containerGroup) assert.NilError(t, err, "no errors should be returned") assert.Equal(t, tc.expectedPodPhase, expectedStatus.Phase, "Pod phase is not as expected as current container group phase") assert.Equal(t, len(tc.expectedPodConditions), len(expectedStatus.Conditions), "Pod conditions are not as expected") diff --git a/pkg/provider/mock_aci_test.go b/pkg/provider/mock_aci_test.go index 147fee53..ae1e530b 100644 --- a/pkg/provider/mock_aci_test.go +++ b/pkg/provider/mock_aci_test.go @@ -3,20 +3,20 @@ package provider import ( "context" - azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" "github.com/virtual-kubelet/azure-aci/pkg/client" "github.com/virtual-kubelet/virtual-kubelet/node/api" ) type CreateContainerGroupFunc func(ctx context.Context, resourceGroup, podNS, podName string, cg *client.ContainerGroupWrapper) error -type GetContainerGroupInfoFunc func(ctx context.Context, resourceGroup, namespace, name, nodeName string) (*azaci.ContainerGroup, error) -type GetContainerGroupListFunc func(ctx context.Context, resourceGroup string) (*[]azaci.ContainerGroup, error) -type ListCapabilitiesFunc func(ctx context.Context, region string) (*[]azaci.Capabilities, error) +type GetContainerGroupInfoFunc func(ctx context.Context, resourceGroup, namespace, name, nodeName string) (*azaciv2.ContainerGroup, error) +type GetContainerGroupListFunc func(ctx context.Context, resourceGroup string) ([]*azaciv2.ContainerGroup, error) +type ListCapabilitiesFunc func(ctx context.Context, region string) ([]*azaciv2.Capabilities, error) type DeleteContainerGroupFunc func(ctx context.Context, resourceGroup, cgName string) error type ListLogsFunc func(ctx context.Context, resourceGroup, cgName, containerName string, opts api.ContainerLogOpts) (*string, error) -type ExecuteContainerCommandFunc func(ctx context.Context, resourceGroup, cgName, containerName string, containerReq azaci.ContainerExecRequest) (azaci.ContainerExecResponse, error) +type ExecuteContainerCommandFunc func(ctx context.Context, resourceGroup, cgName, containerName string, containerReq azaciv2.ContainerExecRequest) (*azaciv2.ContainerExecResponse, error) -type GetContainerGroupFunc func(ctx context.Context, resourceGroup, containerGroupName string) (*client.ContainerGroupWrapper, error) +type GetContainerGroupFunc func(ctx context.Context, resourceGroup, containerGroupName string) (*azaciv2.ContainerGroup, error) type MockACIProvider struct { MockCreateContainerGroup CreateContainerGroupFunc @@ -36,21 +36,21 @@ func NewMockACIProvider(capList ListCapabilitiesFunc) *MockACIProvider { return mock } -func (m *MockACIProvider) ListCapabilities(ctx context.Context, region string) (*[]azaci.Capabilities, error) { +func (m *MockACIProvider) ListCapabilities(ctx context.Context, region string) ([]*azaciv2.Capabilities, error) { if m.MockListCapabilities != nil { return m.MockListCapabilities(ctx, region) } return nil, nil } -func (m *MockACIProvider) GetContainerGroupListResult(ctx context.Context, resourcegroup string) (*[]azaci.ContainerGroup, error) { +func (m *MockACIProvider) GetContainerGroupListResult(ctx context.Context, resourcegroup string) ([]*azaciv2.ContainerGroup, error) { if m.MockGetContainerGroupList != nil { return m.MockGetContainerGroupList(ctx, resourcegroup) } return nil, nil } -func (m *MockACIProvider) GetContainerGroupInfo(ctx context.Context, resourceGroup, namespace, name, nodeName string) (*azaci.ContainerGroup, error) { +func (m *MockACIProvider) GetContainerGroupInfo(ctx context.Context, resourceGroup, namespace, name, nodeName string) (*azaciv2.ContainerGroup, error) { if m.MockGetContainerGroupInfo != nil { return m.MockGetContainerGroupInfo(ctx, resourceGroup, namespace, name, nodeName) } @@ -77,15 +77,15 @@ func (m *MockACIProvider) ListLogs(ctx context.Context, resourceGroup, cgName, c return nil, nil } -func (m *MockACIProvider) ExecuteContainerCommand(ctx context.Context, resourceGroup, cgName, containerName string, containerReq azaci.ContainerExecRequest) (*azaci.ContainerExecResponse, error) { +func (m *MockACIProvider) ExecuteContainerCommand(ctx context.Context, resourceGroup, cgName, containerName string, containerReq azaciv2.ContainerExecRequest) (*azaciv2.ContainerExecResponse, error) { if m.MockExecuteContainerCommand != nil { result, err := m.MockExecuteContainerCommand(ctx, resourceGroup, cgName, containerName, containerReq) - return &result, err + return result, err } return nil, nil } -func (m *MockACIProvider) GetContainerGroup(ctx context.Context, resourceGroup, containerGroupName string) (*client.ContainerGroupWrapper, error) { +func (m *MockACIProvider) GetContainerGroup(ctx context.Context, resourceGroup, containerGroupName string) (*azaciv2.ContainerGroup, error) { if m.MockGetContainerGroup != nil { return m.MockGetContainerGroup(ctx, resourceGroup, containerGroupName) } diff --git a/pkg/tests/utils.go b/pkg/tests/utils.go index 9c7b5166..48bb50fa 100644 --- a/pkg/tests/utils.go +++ b/pkg/tests/utils.go @@ -7,8 +7,8 @@ package tests import ( "time" - azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" - "github.com/Azure/go-autorest/autorest/date" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" + "github.com/virtual-kubelet/azure-aci/pkg/util" v12 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,16 +33,18 @@ var ( testCPU = float64(0.99) testMemory = float64(1.5) port = int32(80) + gpuSKUP100 = azaciv2.GpuSKUP100 + scheme = azaciv2.SchemeHTTP ) -func CreateContainerGroupObj(cgName, cgNamespace, cgState string, containers *[]azaci.Container, provisioningState string) *azaci.ContainerGroup { - fakeIPAddress := azaci.IPAddress{ +func CreateContainerGroupObj(cgName, cgNamespace, cgState string, containers []*azaciv2.Container, provisioningState string) *azaciv2.ContainerGroup { + fakeIPAddress := azaciv2.IPAddress{ IP: &FakeIP, } timeAsString := v1.NewTime(cgCreationTime).String() nodeName := "vk" - return &azaci.ContainerGroup{ + return &azaciv2.ContainerGroup{ Tags: map[string]*string{ "CreationTimestamp": &timeAsString, "PodName": &cgName, @@ -53,9 +55,9 @@ func CreateContainerGroupObj(cgName, cgNamespace, cgState string, containers *[] }, Name: &cgName, ID: &cgName, - ContainerGroupProperties: &azaci.ContainerGroupProperties{ + Properties: &azaciv2.ContainerGroupPropertiesProperties{ Containers: containers, - InstanceView: &azaci.ContainerGroupPropertiesInstanceView{ + InstanceView: &azaciv2.ContainerGroupPropertiesInstanceView{ State: &cgState, }, ProvisioningState: &provisioningState, @@ -64,40 +66,40 @@ func CreateContainerGroupObj(cgName, cgNamespace, cgState string, containers *[] } } -func CreateACIContainersListObj(currentState, PrevState string, startTime, finishTime time.Time, hasResources, hasLimits, hasGPU bool) *[]azaci.Container { - containerList := append([]azaci.Container{}, *CreateACIContainerObj(currentState, PrevState, startTime, finishTime, hasResources, hasLimits, hasGPU)) - return &containerList +func CreateACIContainersListObj(currentState, PrevState string, startTime, finishTime time.Time, hasResources, hasLimits, hasGPU bool) []*azaciv2.Container { + containerList := append([]*azaciv2.Container{}, CreateACIContainerObj(currentState, PrevState, startTime, finishTime, hasResources, hasLimits, hasGPU)) + return containerList } -func CreateACIContainerObj(currentState, PrevState string, startTime, finishTime time.Time, hasResources, hasLimits, hasGPU bool) *azaci.Container { - return &azaci.Container{ +func CreateACIContainerObj(currentState, PrevState string, startTime, finishTime time.Time, hasResources, hasLimits, hasGPU bool) *azaciv2.Container { + return &azaciv2.Container{ Name: &TestContainerName, - ContainerProperties: &azaci.ContainerProperties{ + Properties: &azaciv2.ContainerProperties{ Image: &TestImageNginx, - Ports: &[]azaci.ContainerPort{ + Ports: []*azaciv2.ContainerPort{ { - Protocol: azaci.ContainerNetworkProtocolTCP, + Protocol: &util.ContainerNetworkProtocolTCP, Port: &port, }, }, Resources: CreateContainerResources(hasResources, hasLimits, hasGPU), - Command: &[]string{"nginx", "-g", "daemon off;"}, - InstanceView: &azaci.ContainerPropertiesInstanceView{ + Command: []*string{}, + InstanceView: &azaciv2.ContainerPropertiesInstanceView{ CurrentState: CreateContainerStateObj(currentState, startTime, finishTime, 0), PreviousState: CreateContainerStateObj(PrevState, cgCreationTime, startTime, 0), RestartCount: &RestartCount, - Events: &[]azaci.Event{}, + Events: []*azaciv2.Event{}, }, - LivenessProbe: &azaci.ContainerProbe{}, - ReadinessProbe: &azaci.ContainerProbe{}, + LivenessProbe: &azaciv2.ContainerProbe{}, + ReadinessProbe: &azaciv2.ContainerProbe{}, }, } } -func CreateContainerResources(hasResources, hasLimits, hasGPU bool) *azaci.ResourceRequirements { +func CreateContainerResources(hasResources, hasLimits, hasGPU bool) *azaciv2.ResourceRequirements { if hasResources { - return &azaci.ResourceRequirements{ - Requests: &azaci.ResourceRequests{ + return &azaciv2.ResourceRequirements{ + Requests: &azaciv2.ResourceRequests{ CPU: &testCPU, MemoryInGB: &testMemory, Gpu: CreateGPUResource(hasGPU), @@ -108,9 +110,9 @@ func CreateContainerResources(hasResources, hasLimits, hasGPU bool) *azaci.Resou return nil } -func CreateResourceLimits(hasLimits, hasGPU bool) *azaci.ResourceLimits { +func CreateResourceLimits(hasLimits, hasGPU bool) *azaciv2.ResourceLimits { if hasLimits { - return &azaci.ResourceLimits{ + return &azaciv2.ResourceLimits{ CPU: &testCPU, MemoryInGB: &testMemory, Gpu: CreateGPUResource(hasGPU), @@ -119,30 +121,66 @@ func CreateResourceLimits(hasLimits, hasGPU bool) *azaci.ResourceLimits { return nil } -func CreateGPUResource(hasGPU bool) *azaci.GpuResource { +func CreateGPUResource(hasGPU bool) *azaciv2.GpuResource { if hasGPU { - return &azaci.GpuResource{ + return &azaciv2.GpuResource{ Count: &testGPUCount, - Sku: azaci.GpuSkuP100, + SKU: &gpuSKUP100, } } return nil } -func CreateContainerStateObj(state string, startTime, finishTime time.Time, exitCode int32) *azaci.ContainerState { - return &azaci.ContainerState{ - State: &state, - StartTime: &date.Time{ - Time: startTime, - }, - ExitCode: &exitCode, - FinishTime: &date.Time{ - Time: finishTime, - }, +func CreateContainerStateObj(state string, startTime, finishTime time.Time, exitCode int32) *azaciv2.ContainerState { + return &azaciv2.ContainerState{ + State: &state, + StartTime: &startTime, + ExitCode: &exitCode, + FinishTime: &finishTime, DetailStatus: &emptyStr, } } +func CreateCGProbeObj(hasHTTPGet, hasExec bool) *azaciv2.ContainerProbe { + var bin, c, command, path string + + bin = "/bin/sh" + c = "-c" + command = "/probes/" + path = "/" + port := int32(8080) + fakeNum := int32(0) + + var exec *azaciv2.ContainerExec + var httpGet *azaciv2.ContainerHTTPGet + + if hasExec { + exec = &azaciv2.ContainerExec{ + Command: []*string{ + &bin, + &c, + &command, + }, + } + } + if hasHTTPGet { + httpGet = &azaciv2.ContainerHTTPGet{ + Port: &port, + Path: &path, + Scheme: &scheme, + } + } + return &azaciv2.ContainerProbe{ + Exec: exec, + HTTPGet: httpGet, + InitialDelaySeconds: &fakeNum, + FailureThreshold: &fakeNum, + SuccessThreshold: &fakeNum, + TimeoutSeconds: &fakeNum, + PeriodSeconds: &fakeNum, + } +} + func GetPodConditions(creationTime, readyConditionTime v1.Time, readyConditionStatus v12.ConditionStatus) []v12.PodCondition { return []v12.PodCondition{ { @@ -220,6 +258,44 @@ func CreatePodObj(podName, podNamespace string) *v12.Pod { } } +func CreatePodProbeObj(hasHTTPGet, hasExec bool) *v12.Probe { + var httpGet *v12.HTTPGetAction + var exec *v12.ExecAction + + if hasHTTPGet { + httpGet = &v12.HTTPGetAction{ + Port: intstr.FromString("http"), + Path: "/", + Scheme: "http", + } + } + if hasExec { + exec = &v12.ExecAction{ + Command: []string{ + "/bin/sh", + "-c", + "/probes/", + }, + } + } + + return &v12.Probe{ + Handler: v12.Handler{ + HTTPGet: httpGet, + Exec: exec, + }, + } +} + +func CreateContainerPortObj(portName string, containerPort int32) []v12.ContainerPort { + return []v12.ContainerPort{ + { + Name: portName, + ContainerPort: containerPort, + }, + } +} + func CreatePodVolumeObj(azureFileVolumeName string, fakeSecretName string, projectedVolumeName string) []v12.Volume { emptyVolumeName := "emptyVolumeName" fakeShareName1 := "aksshare1" diff --git a/pkg/util/aci_utils.go b/pkg/util/aci_utils.go index 42a4a244..51e241b6 100644 --- a/pkg/util/aci_utils.go +++ b/pkg/util/aci_utils.go @@ -10,10 +10,23 @@ import ( "fmt" "strings" - "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" v1 "k8s.io/api/core/v1" ) +var ( + // LogTypeContainerInsights to prevent indirect pointer access + LogTypeContainerInsights = azaciv2.LogAnalyticsLogTypeContainerInsights + // ContainerNetworkProtocolTCP to prevent indirect pointer access + ContainerNetworkProtocolTCP = azaciv2.ContainerNetworkProtocolTCP + //ContainerNetworkProtocolUDP to prevent indirect pointer access + ContainerNetworkProtocolUDP = azaciv2.ContainerNetworkProtocolUDP + // ContainerGroupIPAddressTypePublic to prevent indirect pointer access + ContainerGroupIPAddressTypePublic = azaciv2.ContainerGroupIPAddressTypePublic + // ContainerGroupNetworkProtocolTCP to prevent indirect pointer access + ContainerGroupNetworkProtocolTCP = azaciv2.ContainerGroupNetworkProtocolTCP +) + func GetContainerID(cgID, containerName *string) string { if cgID == nil { return "" @@ -42,11 +55,11 @@ func OmitDuplicates(strs []string) []string { return ret } -func GetProtocol(pro v1.Protocol) containerinstance.ContainerNetworkProtocol { +func GetProtocol(pro v1.Protocol) *azaciv2.ContainerNetworkProtocol { switch pro { case v1.ProtocolUDP: - return containerinstance.ContainerNetworkProtocolUDP + return &ContainerNetworkProtocolUDP default: - return containerinstance.ContainerNetworkProtocolTCP + return &ContainerNetworkProtocolTCP } } diff --git a/pkg/validation/validator.go b/pkg/validation/validator.go index e39e0dd7..19329f99 100644 --- a/pkg/validation/validator.go +++ b/pkg/validation/validator.go @@ -1,84 +1,94 @@ package validation import ( - "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance" + "context" + + azaciv2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerinstance/armcontainerinstance/v2" "github.com/pkg/errors" + "github.com/virtual-kubelet/virtual-kubelet/log" ) -func ValidateContainer(container containerinstance.Container) error { - +func ValidateContainer(ctx context.Context, container *azaciv2.Container) error { if container.Name == nil { return errors.Errorf("container name cannot be nil") } - if container.Ports == nil { + if container.Properties.Ports == nil { return errors.Errorf("container %s Ports cannot be nil", *container.Name) } - if container.Image == nil { + if container.Properties.Image == nil { return errors.Errorf("container %s Image cannot be nil", *container.Name) } - if container.ContainerProperties == nil { + if container.Properties == nil { return errors.Errorf("container %s properties cannot be nil", *container.Name) } - if container.InstanceView == nil { + if container.Properties.InstanceView == nil { return errors.Errorf("container %s properties InstanceView cannot be nil", *container.Name) } - if container.InstanceView.CurrentState == nil { + if container.Properties.InstanceView.CurrentState == nil { return errors.Errorf("container %s properties CurrentState cannot be nil", *container.Name) } - if container.InstanceView.CurrentState.StartTime == nil { + if container.Properties.InstanceView.CurrentState.StartTime == nil { return errors.Errorf("container %s properties CurrentState StartTime cannot be nil", *container.Name) } - if container.InstanceView.PreviousState == nil { + if container.Properties.InstanceView.PreviousState == nil { pendingState := "Pending" - container.InstanceView.PreviousState = &containerinstance.ContainerState{ + container.Properties.InstanceView.PreviousState = &azaciv2.ContainerState{ State: &pendingState, DetailStatus: &pendingState, } return nil } - if container.InstanceView.RestartCount == nil { + if container.Properties.InstanceView.RestartCount == nil { return errors.Errorf("container %s properties RestartCount cannot be nil", *container.Name) } - if container.InstanceView.Events == nil { + if container.Properties.InstanceView.Events == nil { return errors.Errorf("container %s properties Events cannot be nil", *container.Name) } - + log.G(ctx).Infof("container %s was validated successfully!", *container.Name) return nil } -func ValidateContainerGroup(cg *containerinstance.ContainerGroup) error { - if cg == nil { - return errors.Errorf("container group cannot be nil") - } +func ValidateContainerGroup(ctx context.Context, cg *azaciv2.ContainerGroup) error { if cg.Name == nil { return errors.Errorf("container group Name cannot be nil") } if cg.ID == nil { return errors.Errorf("container group ID cannot be nil, name: %s", *cg.Name) } - if cg.ContainerGroupProperties == nil { + if cg.Properties == nil { return errors.Errorf("container group properties cannot be nil, name: %s", *cg.Name) } - if cg.Containers == nil { + if cg.Properties.Containers == nil { return errors.Errorf("containers list cannot be nil for container group %s", *cg.Name) } if cg.Tags == nil { return errors.Errorf("tags list cannot be nil for container group %s", *cg.Name) } - if cg.OsType != containerinstance.OperatingSystemTypesWindows { - if cg.IPAddress == nil { + if cg.Properties.InstanceView == nil { + return errors.Errorf("InstanceView cannot be nil for container group %s", *cg.Name) + } + if cg.Properties.InstanceView.State == nil { + return errors.Errorf("InstanceView state cannot be nil for container group %s", *cg.Name) + } + if cg.Properties.OSType != nil && + *cg.Properties.OSType != azaciv2.OperatingSystemTypesWindows { + if cg.Properties.IPAddress == nil { return errors.Errorf("IPAddress cannot be nil for container group %s", *cg.Name) } else { - aciState := *cg.ContainerGroupProperties.ProvisioningState - if cg.IPAddress.IP == nil { + if cg.Properties.ProvisioningState == nil { + return errors.Errorf("ProvisioningState cannot be nil for container group %s", *cg.Name) + } + aciState := *cg.Properties.ProvisioningState + if cg.Properties.IPAddress.IP == nil { if aciState == "Running" { return errors.Errorf("podIP cannot be nil for container group %s while state is %s ", *cg.Name, aciState) } else { emptyIP := "" - cg.IPAddress.IP = &emptyIP + cg.Properties.IPAddress.IP = &emptyIP } } } } + log.G(ctx).Infof("container group %s was validated successfully!", *cg.Name) return nil }