Update how veneur handles aws credentials. (#758)

* Update how veneur handles aws credentials.

* If aws creds are set in config file, try and load those up.
* If no creds are set, follow the aws sdk golang loading order.

* golang→go

Co-authored-by: Allen Sanabria <allen_sanabria@intuit.com>
Co-authored-by: Aditya Mukerjee <dev@chimeracoder.net>
Co-authored-by: Aditya Mukerjee <ChimeraCoder@users.noreply.github.com>

CHANGELOG.md

@@ -34,6 +34,7 @@
 * Updated the vendored version of DataDog/datadog-go which adds support for sending metrics to Unix Domain socket. Thanks, [prudhvi](https://github.com/prudhvi)!
 * Splunk sink: Downgraded Splunk HEC errors to be logged at warning level, rather than error level. Added a note to clarify that Splunk cluster restarts can cause temporary errors, which are not necessarily problematic. Thanks, [aditya](https://github.com/chimeracoder)!
 * Updated the vendored version of github.com/gogo/protobuf which fixes Gopkg.toml conflicts for users of veneur. Thanks, [dtbartle](http://github.com/dtbartle)!
+* Updated server.go to use the aws sdk (https://docs.aws.amazon.com/sdk-for-go/api/aws/session/) when the creds are not set in the config.yaml. Thanks, [linuxdynasty](https://github.com/linuxdynasty)!
 
 ## Bugfixes
 * veneur-prometheus now reports incremental counters instead of cumulative counters. This may cause dramatic differences in the statistics reported by veneur-prometheus.  Thanks, [kklipsch-stripe](https://github.com/kklipsch-stripe)!

plugins/s3/README.md

@@ -4,3 +4,24 @@ S3 Plugin
 The S3 plugin archives every flush to S3 as a separate S3 object.
 
 This plugin is still in an experimental state.
+
+
+
+# Config Options to connect to S3
+
+Mandatory parameters below.
+
+* aws_s3_bucket: `string`
+* aws_region: `string`
+
+Optional parameters below.
+
+* aws_access_key_id `string`
+* aws_secret_access_key `string`
+
+The Go AWS SDK will load up Credentials in the following order. https://docs.aws.amazon.com/sdk-for-go/api/aws/session/
+
+1. Environment Variables `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`, `AWS_PROFILE`, `AWS_REGION`
+2. Shared Credentials file `~/.aws/credentials`
+3. Shared Configuration file (if SharedConfig is enabled) `export AWS_SDK_LOAD_CONFIG=1`
+4. EC2 Instance Metadata (credentials only).

server.go

@@ -684,28 +684,32 @@ func NewFromConfig(logger *logrus.Logger, conf Config) (*Server, error) {
 	awsID := conf.AwsAccessKeyID
 	awsSecret := conf.AwsSecretAccessKey
 	if conf.AwsS3Bucket != "" {
+		var sess *session.Session
+		var err error
 		if len(awsID) > 0 && len(awsSecret) > 0 {
-			sess, err := session.NewSession(&aws.Config{
+			sess, err = session.NewSession(&aws.Config{
 				Region:      aws.String(conf.AwsRegion),
 				Credentials: credentials.NewStaticCredentials(awsID, awsSecret, ""),
 			})
+		} else {
+			sess, err = session.NewSession(&aws.Config{
+				Region: aws.String(conf.AwsRegion),
+			})
+		}
 
-			if err != nil {
-				logger.Infof("error getting AWS session: %s", err)
-				svc = nil
-			} else {
-				logger.Info("Successfully created AWS session")
-				svc = s3.New(sess)
-				plugin := &s3p.S3Plugin{
-					Logger:   log,
-					Svc:      svc,
-					S3Bucket: conf.AwsS3Bucket,
-					Hostname: ret.Hostname,
-				}
-				ret.registerPlugin(plugin)
-			}
+		if err != nil {
+			logger.Infof("error getting AWS session: %s", err)
+			svc = nil
 		} else {
-			logger.Info("AWS S3 credentials not found. S3 plugin is disabled.")
+			logger.Info("Successfully created AWS session")
+			svc = s3.New(sess)
+			plugin := &s3p.S3Plugin{
+				Logger:   log,
+				Svc:      svc,
+				S3Bucket: conf.AwsS3Bucket,
+				Hostname: ret.Hostname,
+			}
+			ret.registerPlugin(plugin)
 		}
 	} else {
 		logger.Info("AWS S3 bucket not set. Skipping S3 Plugin initialization.")