[7.x](backport #26832) Add Proxy settings to AWS Common (#27077)

* Add Proxy settings to AWS Common (#26832) (cherry picked from commit 94af9df) * Update CHANGELOG.next.asciidoc Co-authored-by: Alex Resnick <adr8292@gmail.com> Co-authored-by: kaiyan-sheng <kaiyan.sheng@elastic.co>
elastic · Jul 28, 2021 · 3b755e2 · 3b755e2
1 parent 89f0b66
commit 3b755e2
Show file tree

Hide file tree

Showing 25 changed files with 84 additions and 22 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -466,6 +466,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add orchestrator.cluster.name/url fields as k8s metadata {pull}26056[26056]
 - Libbeat: report beat version to monitoring. {pull}26214[26214]
 - Ensure common proxy settings support in HTTP clients: proxy_disabled, proxy_url, proxy_headers and typical environment variables HTTP_PROXY, HTTPS_PROXY, NOPROXY. {pull}25219[25219]
+- Add proxy support for AWS functions. {pull}26832[26832]
 
 *Auditbeat*
 

diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc
@@ -53,6 +53,7 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
     #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+    #var.proxy_url: http://proxy:8080
 
   cloudwatch:
     enabled: false
@@ -66,6 +67,7 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
     #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+    #var.proxy_url: http://proxy:8080
 
   ec2:
     enabled: false
@@ -79,6 +81,7 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
     #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+    #var.proxy_url: http://proxy:8080
 
   elb:
     enabled: false
@@ -92,6 +95,7 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
     #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+    #var.proxy_url: http://proxy:8080
 
   s3access:
     enabled: false
@@ -105,6 +109,7 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
     #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+    #var.proxy_url: http://proxy:8080
 
   vpcflow:
     enabled: false
@@ -118,6 +123,7 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
     #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+    #var.proxy_url: http://proxy:8080
 ----
 
 *`var.queue_url`*::

diff --git a/libbeat/common/transport/httpcommon/proxy.go b/libbeat/common/transport/httpcommon/proxy.go
@@ -28,7 +28,7 @@ import (
 //
 // Proxy usage will be disabled in general if Disable is set.
 // If URL is not set, the proxy configuration will default
-// to HTTP_PROXY, HTTPS_PPROXY, and NO_PROXY.
+// to HTTP_PROXY, HTTPS_PROXY, and NO_PROXY.
 //
 // The default (and zero) value of HTTPClientProxySettings has Proxy support
 // enabled, and will select the proxy per URL based on the environment variables.

diff --git a/x-pack/filebeat/input/awscloudwatch/input.go b/x-pack/filebeat/input/awscloudwatch/input.go
@@ -103,9 +103,9 @@ func NewInput(cfg *common.Config, connector channel.Connector, context input.Con
 		config.RegionName = regionName
 	}
 
-	awsConfig, err := awscommon.GetAWSCredentials(config.AwsConfig)
+	awsConfig, err := awscommon.InitializeAWSConfig(config.AwsConfig)
 	if err != nil {
-		return nil, errors.Wrap(err, "getAWSCredentials failed")
+		return nil, errors.Wrap(err, "InitializeAWSConfig failed")
 	}
 	awsConfig.Region = config.RegionName
 

diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go
@@ -54,9 +54,9 @@ func newInput(config config) (*s3Input, error) {
 func (in *s3Input) Name() string { return inputName }
 
 func (in *s3Input) Test(ctx v2.TestContext) error {
-	_, err := awscommon.GetAWSCredentials(in.config.AWSConfig)
+	_, err := awscommon.InitializeAWSConfig(in.config.AWSConfig)
 	if err != nil {
-		return fmt.Errorf("getAWSCredentials failed: %w", err)
+		return fmt.Errorf("InitializeAWSConfig failed: %w", err)
 	}
 	return nil
 }
@@ -98,9 +98,9 @@ func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3C
 		log = log.With("region", regionName)
 	}
 
-	awsConfig, err := awscommon.GetAWSCredentials(in.config.AWSConfig)
+	awsConfig, err := awscommon.InitializeAWSConfig(in.config.AWSConfig)
 	if err != nil {
-		return nil, fmt.Errorf("getAWSCredentials failed: %w", err)
+		return nil, fmt.Errorf("InitializeAWSConfig failed: %w", err)
 	}
 	awsConfig.Region = regionName
 

diff --git a/x-pack/filebeat/input/awss3/s3_integration_test.go b/x-pack/filebeat/input/awss3/s3_integration_test.go
@@ -138,9 +138,9 @@ func setupCollector(t *testing.T, cfg *common.Config, mock bool) (*s3Collector,
 	}
 
 	config := getConfigForTest(t)
-	awsConfig, err := awscommon.GetAWSCredentials(config.AWSConfig)
+	awsConfig, err := awscommon.InitializeAWSConfig(config.AWSConfig)
 	if err != nil {
-		t.Fatal("failed GetAWSCredentials with AWS Config: ", err)
+		t.Fatal("failed InitializeAWSConfig with AWS Config: ", err)
 	}
 
 	s3BucketRegion := os.Getenv("S3_BUCKET_REGION")

diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc
@@ -48,6 +48,7 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
     #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+    #var.proxy_url: http://proxy:8080
 
   cloudwatch:
     enabled: false
@@ -61,6 +62,7 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
     #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+    #var.proxy_url: http://proxy:8080
 
   ec2:
     enabled: false
@@ -74,6 +76,7 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
     #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+    #var.proxy_url: http://proxy:8080
 
   elb:
     enabled: false
@@ -87,6 +90,7 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
     #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+    #var.proxy_url: http://proxy:8080
 
   s3access:
     enabled: false
@@ -100,6 +104,7 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
     #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+    #var.proxy_url: http://proxy:8080
 
   vpcflow:
     enabled: false
@@ -113,6 +118,7 @@ Example config:
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
     #var.role_arn: arn:aws:iam::123456789012:role/test-mb
+    #var.proxy_url: http://proxy:8080
 ----
 
 *`var.queue_url`*::

diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml
@@ -59,6 +59,10 @@ fips_enabled: {{ .fips_enabled }}
 max_number_of_messages: {{ .max_number_of_messages }}
 {{ end }}
 
+{{ if .proxy_url }}
+proxy_url: {{ .proxy_url }}
+{{ end }}
+
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 

diff --git a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml
@@ -22,6 +22,7 @@ var:
   - name: process_insight_logs
     default: true
   - name: fips_enabled
+  - name: proxy_url
   - name: max_number_of_messages
 
 ingest_pipeline: ingest/pipeline.yml

diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml b/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml
@@ -45,6 +45,10 @@ fips_enabled: {{ .fips_enabled }}
 max_number_of_messages: {{ .max_number_of_messages }}
 {{ end }}
 
+{{ if .proxy_url }}
+proxy_url: {{ .proxy_url }}
+{{ end }}
+
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 

diff --git a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml
@@ -16,6 +16,7 @@ var:
   - name: tags
     default: [forwarded]
   - name: fips_enabled
+  - name: proxy_url
   - name: max_number_of_messages
 
 ingest_pipeline: ingest/pipeline.yml

diff --git a/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml b/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml
@@ -45,6 +45,10 @@ fips_enabled: {{ .fips_enabled }}
 max_number_of_messages: {{ .max_number_of_messages }}
 {{ end }}
 
+{{ if .proxy_url }}
+proxy_url: {{ .proxy_url }}
+{{ end }}
+
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 

diff --git a/x-pack/filebeat/module/aws/ec2/manifest.yml b/x-pack/filebeat/module/aws/ec2/manifest.yml
@@ -16,6 +16,7 @@ var:
   - name: tags
     default: [forwarded]
   - name: fips_enabled
+  - name: proxy_url
   - name: max_number_of_messages
 
 ingest_pipeline: ingest/pipeline.yml

diff --git a/x-pack/filebeat/module/aws/elb/config/aws-s3.yml b/x-pack/filebeat/module/aws/elb/config/aws-s3.yml
@@ -45,6 +45,10 @@ fips_enabled: {{ .fips_enabled }}
 max_number_of_messages: {{ .max_number_of_messages }}
 {{ end }}
 
+{{ if .proxy_url }}
+proxy_url: {{ .proxy_url }}
+{{ end }}
+
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 

diff --git a/x-pack/filebeat/module/aws/elb/manifest.yml b/x-pack/filebeat/module/aws/elb/manifest.yml
@@ -16,6 +16,7 @@ var:
   - name: tags
     default: [forwarded]
   - name: fips_enabled
+  - name: proxy_url
   - name: max_number_of_messages
 
 ingest_pipeline: ingest/pipeline.yml

diff --git a/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml b/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml
@@ -45,6 +45,10 @@ fips_enabled: {{ .fips_enabled }}
 max_number_of_messages: {{ .max_number_of_messages }}
 {{ end }}
 
+{{ if .proxy_url }}
+proxy_url: {{ .proxy_url }}
+{{ end }}
+
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 

diff --git a/x-pack/filebeat/module/aws/s3access/manifest.yml b/x-pack/filebeat/module/aws/s3access/manifest.yml
@@ -16,6 +16,7 @@ var:
   - name: tags
     default: [forwarded]
   - name: fips_enabled
+  - name: proxy_url
   - name: max_number_of_messages
 
 ingest_pipeline: ingest/pipeline.yml

diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml
@@ -47,6 +47,10 @@ fips_enabled: {{ .fips_enabled }}
 max_number_of_messages: {{ .max_number_of_messages }}
 {{ end }}
 
+{{ if .proxy_url }}
+proxy_url: {{ .proxy_url }}
+{{ end }}
+
 {{ else if eq .input "file" }}
 
 type: log

diff --git a/x-pack/filebeat/module/aws/vpcflow/manifest.yml b/x-pack/filebeat/module/aws/vpcflow/manifest.yml
@@ -16,6 +16,7 @@ var:
   - name: tags
     default: [forwarded]
   - name: fips_enabled
+  - name: proxy_url
   - name: max_number_of_messages
 
 ingest_pipeline: ingest/pipeline.yml

diff --git a/x-pack/functionbeat/manager/aws/cli_manager.go b/x-pack/functionbeat/manager/aws/cli_manager.go
@@ -214,7 +214,7 @@ func NewCLI(
 	if err := cfg.Unpack(config); err != nil {
 		return nil, err
 	}
-	awsCfg, err := awscommon.GetAWSCredentials(config.Credentials)
+	awsCfg, err := awscommon.InitializeAWSConfig(config.Credentials)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get aws credentials, please check AWS credential in config: %+v", err)
 	}

diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go
@@ -52,7 +52,7 @@ func AutodiscoverBuilder(
 		return nil, err
 	}
 
-	awsCfg, err := awscommon.GetAWSCredentials(
+	awsCfg, err := awscommon.InitializeAWSConfig(
 		awscommon.ConfigAWS{
 			AccessKeyID:     config.AWSConfig.AccessKeyID,
 			SecretAccessKey: config.AWSConfig.SecretAccessKey,

diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go
@@ -54,7 +54,7 @@ func AutodiscoverBuilder(
 		return nil, err
 	}
 
-	awsCfg, err := awscommon.GetAWSCredentials(awscommon.ConfigAWS{
+	awsCfg, err := awscommon.InitializeAWSConfig(awscommon.ConfigAWS{
 		AccessKeyID:     config.AWSConfig.AccessKeyID,
 		SecretAccessKey: config.AWSConfig.SecretAccessKey,
 		SessionToken:    config.AWSConfig.SessionToken,
@@ -76,7 +76,7 @@ func AutodiscoverBuilder(
 
 	var clients []elasticloadbalancingv2iface.ClientAPI
 	for _, region := range config.Regions {
-		awsCfg, err := awscommon.GetAWSCredentials(awscommon.ConfigAWS{
+		awsCfg, err := awscommon.InitializeAWSConfig(awscommon.ConfigAWS{
 			AccessKeyID:     config.AWSConfig.AccessKeyID,
 			SecretAccessKey: config.AWSConfig.SecretAccessKey,
 			SessionToken:    config.AWSConfig.SessionToken,

diff --git a/x-pack/libbeat/common/aws/credentials.go b/x-pack/libbeat/common/aws/credentials.go
@@ -5,6 +5,9 @@
 package aws
 
 import (
+	"net/http"
+	"net/url"
+
 	awssdk "github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/aws/defaults"
 	"github.com/aws/aws-sdk-go-v2/aws/external"
@@ -18,14 +21,29 @@ import (
 
 // ConfigAWS is a structure defined for AWS credentials
 type ConfigAWS struct {
-	AccessKeyID          string `config:"access_key_id"`
-	SecretAccessKey      string `config:"secret_access_key"`
-	SessionToken         string `config:"session_token"`
-	ProfileName          string `config:"credential_profile_name"`
-	SharedCredentialFile string `config:"shared_credential_file"`
-	Endpoint             string `config:"endpoint"`
-	RoleArn              string `config:"role_arn"`
-	AWSPartition         string `config:"aws_partition"` // Deprecated.
+	AccessKeyID          string   `config:"access_key_id"`
+	SecretAccessKey      string   `config:"secret_access_key"`
+	SessionToken         string   `config:"session_token"`
+	ProfileName          string   `config:"credential_profile_name"`
+	SharedCredentialFile string   `config:"shared_credential_file"`
+	Endpoint             string   `config:"endpoint"`
+	RoleArn              string   `config:"role_arn"`
+	AWSPartition         string   `config:"aws_partition"` // Deprecated.
+	ProxyUrl             *url.URL `config:"proxy_url"`
+}
+
+// InitializeAWSConfig function creates the awssdk.Config object from the provided config
+func InitializeAWSConfig(config ConfigAWS) (awssdk.Config, error) {
+	AWSConfig, _ := GetAWSCredentials(config)
+	if config.ProxyUrl != nil {
+		httpClient := &http.Client{
+			Transport: &http.Transport{
+				Proxy: http.ProxyURL(config.ProxyUrl),
+			},
+		}
+		AWSConfig.HTTPClient = httpClient
+	}
+	return AWSConfig, nil
 }
 
 // GetAWSCredentials function gets aws credentials from the config.

diff --git a/x-pack/libbeat/docs/aws-credentials-config.asciidoc b/x-pack/libbeat/docs/aws-credentials-config.asciidoc
@@ -17,6 +17,7 @@ Some services, such as IAM, do not support regions. The endpoints for these
 services do not include a region. In `aws` module, `endpoint` config is to set
 the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`,
 `sc2s.sgov.gov`.
+* *proxy_url*: URL of the proxy to use to connect to AWS web services. The syntax is http(s)://<IP/Hostname>:<port>
 
 [float]
 ==== Supported Formats

diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go
@@ -76,7 +76,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 		return nil, err
 	}
 
-	awsConfig, err := awscommon.GetAWSCredentials(config.AWSConfig)
+	awsConfig, err := awscommon.InitializeAWSConfig(config.AWSConfig)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get aws credentials, please check AWS credential in config: %w", err)
 	}