From ae1b974e68587a1aeba01f4a74ae50fd8a2ede2e Mon Sep 17 00:00:00 2001 From: Mahmoud Saada Date: Sat, 7 Oct 2017 09:48:27 -0400 Subject: [PATCH 1/4] Add Azure VM Cloud Metadata Support --- .../add_cloud_metadata/add_cloud_metadata.go | 5 ++ .../add_cloud_metadata/provider_azure_vm.go | 25 ++++++ .../provider_azure_vm_test.go | 77 +++++++++++++++++++ 3 files changed, 107 insertions(+) create mode 100644 libbeat/processors/add_cloud_metadata/provider_azure_vm.go create mode 100644 libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go diff --git a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go index 5b1424fa029..9d55e3c99f6 100644 --- a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go +++ b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go @@ -260,6 +260,10 @@ func setupFetchers(c *common.Config) ([]*metadataFetcher, error) { if err != nil { return fetchers, err } + azFetcher, err := newAzureVmMetadataFetcher(c) + if err != nil { + return fetchers, err + } fetchers = []*metadataFetcher{ doFetcher, @@ -267,6 +271,7 @@ func setupFetchers(c *common.Config) ([]*metadataFetcher, error) { gceFetcher, qcloudFetcher, ecsFetcher, + azFetcher, } return fetchers, nil } diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go new file mode 100644 index 00000000000..10193d9cd4c --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go @@ -0,0 +1,25 @@ +package add_cloud_metadata + +import ( + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstriface" +) + +// Azure VM Metadata Service +func newAzureVmMetadataFetcher(config *common.Config) (*metadataFetcher, error) { + azMetadataURI := "/metadata/instance/compute?api-version=2017-04-02" + azHeaders := map[string]string{"Metadata": "true"} + azSchema := func(m map[string]interface{}) common.MapStr { + out, _ := s.Schema{ + "instance_id": c.Str("vmId"), + "instance_name": c.Str("name"), + "machine_type": c.Str("vmSize"), + "region": c.Str("location"), + }.Apply(m) + return out + } + + fetcher, err := newMetadataFetcher(config, "az", azHeaders, metadataHost, azSchema, azMetadataURI) + return fetcher, err +} diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go new file mode 100644 index 00000000000..ddfe6590357 --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go @@ -0,0 +1,77 @@ +package add_cloud_metadata + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" +) + +const azInstanceIdentityDocument = `{ + "location": "eastus2", + "name": "test-az-vm", + "offer": "UbuntuServer", + "osType": "Linux", + "platformFaultDomain": "0", + "platformUpdateDomain": "0", + "publisher": "Canonical", + "sku": "14.04.4-LTS", + "version": "14.04.201605091", + "vmId": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e", + "vmSize": "Standard_D3_v2" +}` + +func initAzureTestServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/metadata/instance/compute?api-version=2017-04-02" && r.Header.Get("Metadata") == "true" { + w.Write([]byte(azInstanceIdentityDocument)) + return + } + + http.Error(w, "not found", http.StatusNotFound) + })) +} + +func TestRetrieveAzureMetadata(t *testing.T) { + if testing.Verbose() { + logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"}) + } + + server := initAzureTestServer() + defer server.Close() + + config, err := common.NewConfigFrom(map[string]interface{}{ + "host": server.Listener.Addr().String(), + }) + if err != nil { + t.Fatal(err) + } + + p, err := newCloudMetadata(config) + if err != nil { + t.Fatal(err) + } + + actual, err := p.Run(&beat.Event{Fields: common.MapStr{}}) + if err != nil { + t.Fatal(err) + } + + expected := common.MapStr{ + "meta": common.MapStr{ + "cloud": common.MapStr{ + "provider": "az", + "instance_id": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e", + "instance_name": "test-az-vm", + "machine_type": "Standard_D3_v2", + "region": "eastus2", + }, + }, + } + assert.Equal(t, expected, actual.Fields) +} From d50e5aaeb4d2eaa6b2adebdfc8c6083a33074062 Mon Sep 17 00:00:00 2001 From: Mahmoud Saada Date: Mon, 9 Oct 2017 13:52:08 -0400 Subject: [PATCH 2/4] Update CHANGELOG.asciidoc --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index ef534fa6fc6..1bde5134247 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -61,6 +61,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Changed the hashbang used in the beat helper script from `/bin/bash` to `/usr/bin/env bash`. {pull}5051[5051] - Changed beat helper script to use `exec` when running the beat. {pull}5051[5051] - Fix reloader error message to only print on actual error {pull}5066[5066] +- Add Azure VM support for add_cloud_metadata processor {pull}5355[5355] *Auditbeat* From fcd7b2c365e66a92301ce4276b8c8f7f6bfbfd01 Mon Sep 17 00:00:00 2001 From: Mahmoud Saada Date: Mon, 9 Oct 2017 14:02:38 -0400 Subject: [PATCH 3/4] Update processors-using.asciidoc --- libbeat/docs/processors-using.asciidoc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index 22264095793..bcee6c347e5 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -260,6 +260,7 @@ The following cloud providers are supported: - Google Compute Engine (GCE) - https://www.qcloud.com/?lang=en[Tencent Cloud] (QCloud) - Alibaba Cloud (ECS) +- Azure Virtual Machine The simple configuration below enables the processor. @@ -365,6 +366,24 @@ ECS instance. } ------------------------------------------------------------------------------- +_Azure Virtual Machine_ + +[source,json] +------------------------------------------------------------------------------- +{ + "meta": { + "cloud": { + "provider": "az", + "instance_id": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e", + "instance_name": "test-az-vm", + "machine_type": "Standard_D3_v2", + "region": "eastus2" + } + } +} +------------------------------------------------------------------------------- + + [[add-locale]] === Add the local time zone From df068be528d7437e4f06f529d76298c609449d2c Mon Sep 17 00:00:00 2001 From: Mahmoud Saada Date: Mon, 9 Oct 2017 14:05:05 -0400 Subject: [PATCH 4/4] Update processors-using.asciidoc --- libbeat/docs/processors-using.asciidoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index bcee6c347e5..eca64a6715f 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -373,11 +373,11 @@ _Azure Virtual Machine_ { "meta": { "cloud": { - "provider": "az", - "instance_id": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e", - "instance_name": "test-az-vm", - "machine_type": "Standard_D3_v2", - "region": "eastus2" + "provider": "az", + "instance_id": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e", + "instance_name": "test-az-vm", + "machine_type": "Standard_D3_v2", + "region": "eastus2" } } }