Rename google-pubsub Filebeat input to gcp-pubsub (#22213)

* Rename google-pubsub Filebeat input to gcp-pubsub * change to gcp-pubsub input name in elastic-agent * add warning log message for deprecated old input name
elastic · Nov 9, 2020 · 42428a8 · 42428a8
1 parent 0bab1d3
commit 42428a8
Show file tree

Hide file tree

Showing 21 changed files with 51 additions and 32 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -707,6 +707,7 @@ from being added to events by default. {pull}18159[18159]
 - Add SSL option to checkpoint module {pull}19560[19560]
 - Add max_number_of_messages config into s3 input. {pull}21993[21993]
 - Update Okta documentation for new stateful restarts. {pull}22091[22091]
+- Rename google-pubsub input to gcp-pubsub. {pull}22213[22213]
 - Copy tag names from MISP data into events. {pull}21664[21664]
 - Added DNS response IP addresses to `related.ip` in Suricata module. {pull}22291[22291]
 - Added TLS JA3 fingerprint, certificate not_before/not_after, certificate SHA1 hash, and certificate subject fields to Zeek SSL dataset. {pull}21696[21696]

diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc
@@ -66,7 +66,7 @@ You can configure {beatname_uc} to use the following inputs:
 * <<{beatname_lc}-input-cloudfoundry>>
 * <<{beatname_lc}-input-container>>
 * <<{beatname_lc}-input-docker>>
-* <<{beatname_lc}-input-google-pubsub>>
+* <<{beatname_lc}-input-gcp-pubsub>>
 * <<{beatname_lc}-input-http_endpoint>>
 * <<{beatname_lc}-input-httpjson>>
 * <<{beatname_lc}-input-kafka>>
@@ -96,7 +96,7 @@ include::inputs/input-docker.asciidoc[]
 
 include::inputs/input-filestream.asciidoc[]
 
-include::../../x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc[]
+include::../../x-pack/filebeat/docs/inputs/input-gcp-pubsub.asciidoc[]
 
 include::../../x-pack/filebeat/docs/inputs/input-http-endpoint.asciidoc[]
 

diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go
diff --git a/x-pack/elastic-agent/spec/filebeat.yml b/x-pack/elastic-agent/spec/filebeat.yml
@@ -64,7 +64,7 @@ rules:
  - cloudfoundry
  - container
  - docker
- - google-pubsub
+ - gcp-pubsub
  - http_endpoint
  - httpjson
  - kafka

diff --git a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl
@@ -30,7 +30,7 @@
 
 #---------------------------- Google Cloud Pub/Sub Input -----------------------
 # Input for reading messages from a Google Cloud Pub/Sub topic subscription.
-- type: google-pubsub
+- type: gcp-pubsub
  enabled: false
 
  # Google Cloud project ID. Required.

diff --git a/.../docs/inputs/input-google-pubsub.asciidoc → ...eat/docs/inputs/input-gcp-pubsub.asciidoc b/.../docs/inputs/input-google-pubsub.asciidoc → ...eat/docs/inputs/input-gcp-pubsub.asciidoc
@@ -1,15 +1,15 @@
 [role="xpack"]
 
-:type: google-pubsub
+:type: gcp-pubsub
 
 [id="{beatname_lc}-input-{type}"]
-=== Google Cloud Pub/Sub input
+=== GCP Pub/Sub input
 
 ++++
-<titleabbrev>Google Pub/Sub</titleabbrev>
+<titleabbrev>GCP Pub/Sub</titleabbrev>
 ++++
 
-Use the `google-pubsub` input to read messages from a Google Cloud Pub/Sub topic
+Use the `gcp-pubsub` input to read messages from a Google Cloud Pub/Sub topic
 subscription.
 
 This input can, for example, be used to receive Stackdriver logs that have been
@@ -23,7 +23,7 @@ Example configuration:
 ["source","yaml",subs="attributes"]
 ----
 {beatname_lc}.inputs:
-- type: google-pubsub
+- type: gcp-pubsub
  project_id: my-gcp-project-id
  topic: vpc-firewall-logs-topic
  subscription.name: filebeat-vpc-firewall-logs-sub
@@ -33,7 +33,7 @@ Example configuration:
 
 ==== Configuration options
 
-The `google-pubsub` input supports the following configuration options plus the
+The `gcp-pubsub` input supports the following configuration options plus the
 <<{beatname_lc}-input-{type}-common-options>> described later.
 
 [float]

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -2388,7 +2388,7 @@ filebeat.inputs:
 
 #---------------------------- Google Cloud Pub/Sub Input -----------------------
 # Input for reading messages from a Google Cloud Pub/Sub topic subscription.
-- type: google-pubsub
+- type: gcp-pubsub
  enabled: false
 
  # Google Cloud project ID. Required.

diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go
diff --git a/...ebeat/input/googlepubsub/_meta/Dockerfile → ...filebeat/input/gcppubsub/_meta/Dockerfile b/...ebeat/input/googlepubsub/_meta/Dockerfile → ...filebeat/input/gcppubsub/_meta/Dockerfile
diff --git a/...googlepubsub/_meta/supported-versions.yml → ...ut/gcppubsub/_meta/supported-versions.yml b/...googlepubsub/_meta/supported-versions.yml → ...ut/gcppubsub/_meta/supported-versions.yml
diff --git a/x-pack/filebeat/input/googlepubsub/config.go → x-pack/filebeat/input/gcppubsub/config.go b/x-pack/filebeat/input/googlepubsub/config.go → x-pack/filebeat/input/gcppubsub/config.go
@@ -2,18 +2,21 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package googlepubsub
+package gcppubsub
 
 import (
  "context"
  "fmt"
  "os"
 
+ "github.com/elastic/beats/v7/filebeat/harvester"
+
  "cloud.google.com/go/pubsub"
  "golang.org/x/oauth2/google"
 )
 
 type config struct {
+ harvester.ForwarderConfig `config:",inline"`
  // Google Cloud project name.
  ProjectID string `config:"project_id" validate:"required"`
 
@@ -62,6 +65,9 @@ func (c *config) Validate() error {
 
 func defaultConfig() config {
  var c config
+ c.ForwarderConfig = harvester.ForwarderConfig{
+ Type: "gcp-pubsub",
+ }
  c.Subscription.NumGoroutines = 1
  c.Subscription.MaxOutstandingMessages = 1000
  c.Subscription.Create = true

diff --git a/...ilebeat/input/googlepubsub/config_test.go → ...k/filebeat/input/gcppubsub/config_test.go b/...ilebeat/input/googlepubsub/config_test.go → ...k/filebeat/input/gcppubsub/config_test.go
@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package googlepubsub
+package gcppubsub
 
 import (
  "os"

diff --git a/...eat/input/googlepubsub/docker-compose.yml → ...lebeat/input/gcppubsub/docker-compose.yml b/...eat/input/googlepubsub/docker-compose.yml → ...lebeat/input/gcppubsub/docker-compose.yml
diff --git a/x-pack/filebeat/input/googlepubsub/input.go → x-pack/filebeat/input/gcppubsub/input.go b/x-pack/filebeat/input/googlepubsub/input.go → x-pack/filebeat/input/gcppubsub/input.go
@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package googlepubsub
+package gcppubsub
 
 import (
  "context"
@@ -26,14 +26,20 @@ import (
 )
 
 const (
- inputName = "google-pubsub"
+ inputName = "gcp-pubsub"
+ oldInputName = "google-pubsub"
 )
 
 func init() {
  err := input.Register(inputName, NewInput)
  if err != nil {
  panic(errors.Wrapf(err, "failed to register %v input", inputName))
  }
+
+ err = input.Register(oldInputName, NewInput)
+ if err != nil {
+ panic(errors.Wrapf(err, "failed to register %v input", oldInputName))
+ }
 }
 
 type pubsubInput struct {
@@ -64,6 +70,15 @@ func NewInput(
  return nil, err
  }
 
+ logger := logp.NewLogger("gcp.pubsub").With(
+ "pubsub_project", conf.ProjectID,
+ "pubsub_topic", conf.Topic,
+ "pubsub_subscription", conf.Subscription)
+
+ if conf.Type == oldInputName {
+ logger.Warnf("%s input name is deprecated, please use %s instead", oldInputName, inputName)
+ }
+
  // Wrap input.Context's Done channel with a context.Context. This goroutine
  // stops with the parent closes the Done channel.
  inputCtx, cancelInputCtx := context.WithCancel(context.Background())
@@ -80,11 +95,8 @@ func NewInput(
  workerCtx, workerCancel := context.WithCancel(inputCtx)
 
  in := &pubsubInput{
- config: conf,
- log: logp.NewLogger("google.pubsub").With(
- "pubsub_project", conf.ProjectID,
- "pubsub_topic", conf.Topic,
- "pubsub_subscription", conf.Subscription),
+ config: conf,
+ log: logger,
  inputCtx: inputCtx,
  workerCtx: workerCtx,
  workerCancel: workerCancel,
@@ -109,7 +121,7 @@ func NewInput(
  if err != nil {
  return nil, err
  }
- in.log.Info("Initialized Google Pub/Sub input.")
+ in.log.Info("Initialized GCP Pub/Sub input.")
  return in, nil
 }
 

diff --git a/...ilebeat/input/googlepubsub/pubsub_test.go → ...k/filebeat/input/gcppubsub/pubsub_test.go b/...ilebeat/input/googlepubsub/pubsub_test.go → ...k/filebeat/input/gcppubsub/pubsub_test.go
@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package googlepubsub
+package gcppubsub
 
 import (
  "context"

diff --git a/...eat/input/googlepubsub/testdata/fake.json → ...lebeat/input/gcppubsub/testdata/fake.json b/...eat/input/googlepubsub/testdata/fake.json → ...lebeat/input/gcppubsub/testdata/fake.json
diff --git a/x-pack/filebeat/module/googlecloud/audit/config/input.yml b/x-pack/filebeat/module/googlecloud/audit/config/input.yml
@@ -1,6 +1,6 @@
-{{ if eq .input "google-pubsub" }}
+{{ if eq .input "gcp-pubsub" }}
 
-type: google-pubsub
+type: gcp-pubsub
 project_id: {{ .project_id }}
 topic: {{ .topic }}
 subscription.name: {{ .subscription_name }}

diff --git a/x-pack/filebeat/module/googlecloud/audit/manifest.yml b/x-pack/filebeat/module/googlecloud/audit/manifest.yml
@@ -2,7 +2,7 @@ module_version: "1.0"
 
 var:
  - name: input
- default: google-pubsub
+ default: gcp-pubsub
  - name: project_id
  default: SET_PROJECT_NAME
  - name: topic

diff --git a/x-pack/filebeat/module/googlecloud/firewall/config/input.yml b/x-pack/filebeat/module/googlecloud/firewall/config/input.yml
@@ -1,6 +1,6 @@
-{{ if eq .input "google-pubsub" }}
+{{ if eq .input "gcp-pubsub" }}
 
-type: google-pubsub
+type: gcp-pubsub
 project_id: {{ .project_id }}
 topic: {{ .topic }}
 subscription.name: {{ .subscription_name }}

diff --git a/x-pack/filebeat/module/googlecloud/firewall/manifest.yml b/x-pack/filebeat/module/googlecloud/firewall/manifest.yml
@@ -2,7 +2,7 @@ module_version: "1.0"
 
 var:
  - name: input
- default: google-pubsub
+ default: gcp-pubsub
  - name: project_id
  default: SET_PROJECT_NAME
  - name: topic

diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml b/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml
@@ -1,6 +1,6 @@
-{{ if eq .input "google-pubsub" }}
+{{ if eq .input "gcp-pubsub" }}
 
-type: google-pubsub
+type: gcp-pubsub
 project_id: {{ .project_id }}
 topic: {{ .topic }}
 subscription.name: {{ .subscription_name }}