[0.22] Backport of PRs 4815 5321 5444 (#1309)

* [0.22] Backport of PRs 5321 and 4815 (from #1284)

* Backport of (knative#4815)

Extension point for transport used to send events in upgrade tests

* EventSender interface to enable customisation of sending events
* Clean ups
* Basic extendability of wathola sender
* Updating boilerplate
* Testing sender services
* Allow of customization of wathola test images on downstream
* Waiting until test port is open
* Switch to v2 of pelletier/toml to support reading string/map[string]interface{} for interface{} typed field.
* LogLevel is settable
* Logging received event
* Changes after review

* Backport of (knative#5321)

Allow to change SUT for upgrade tests

* Introduction of SUT package
* Refactor to keep used interfaces and deprecate them first
* Update-codegen
* Restoring configuration options, to have Deprecated status instead
* Working e2e test
* Fixing NPE in upgrade tests
* Introducing new interfaces for configuration and easy execution
* NPE fix in continual.go@55
* Removal of ensure.NoError
* Raising UnavailablePeriodToReport to 10s to make tests more stable
* Running all e2e tagged tests
* Raising unavailibility period to 60s as load & retries makes it longer.
* Execute upgrade e2e tests with additional run
* Using SutURL in wathola config instead of BrokerURL
* Removal of redundant namespace option
* Switch to using interface{} as endpoint address representation
* Fixing compile error on prober
* Use full event type only
* Remove unused event prefix
* Renaming after code review
* Removal of unnecessary github.com/prometheus/common
* Remove Kafka from README and SUT to graph
* Remove unused EventsTypePrefix option
* Remove execution of TestBrokerAndTriggers helper test

* Backport of knative#5444 (#1286)

* Don't wait on triggers, before deploying their subscriber

* Restore manually modified vendor dir

go.mod

@@ -15,13 +15,13 @@ require (
  github.com/kelseyhightower/envconfig v1.4.0
  github.com/mitchellh/go-homedir v1.1.0
  github.com/openzipkin/zipkin-go v0.2.5
- github.com/pelletier/go-toml v1.8.0
+ github.com/pelletier/go-toml/v2 v2.0.0-beta.2
  github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
  github.com/pkg/errors v0.9.1
  github.com/rickb777/date v1.13.0
  github.com/robfig/cron/v3 v3.0.1
  github.com/rogpeppe/fastuuid v1.2.0
- github.com/stretchr/testify v1.6.1
+ github.com/stretchr/testify v1.7.0
  github.com/tsenart/vegeta/v12 v12.8.4
  github.com/wavesoftware/go-ensure v1.0.0
  go.opencensus.io v0.23.0

test/lib/resources/eventing.go

@@ -379,43 +379,61 @@ func WithDependencyAnnotationTriggerV1Beta1(dependencyAnnotation string) Trigger
 
 // WithSubscriberServiceRefForTriggerV1Beta1 returns an option that adds a Subscriber Knative Service Ref for the given v1beta1 Trigger.
 func WithSubscriberServiceRefForTriggerV1Beta1(name string) TriggerOptionV1Beta1 {
- return func(t *eventingv1beta1.Trigger) {
- if name != "" {
- t.Spec.Subscriber = duckv1.Destination{
- Ref: KnativeRefForService(name, t.Namespace),
- }
+ return WithSubscriberDestinationV1Beta1(func(t *eventingv1beta1.Trigger) duckv1.Destination {
+ return duckv1.Destination{
+ Ref: KnativeRefForService(name, t.Namespace),
  }
- }
+ })
 }
 
 // WithSubscriberServiceRefForTriggerV1 returns an option that adds a Subscriber Knative Service Ref for the given v1 Trigger.
 func WithSubscriberServiceRefForTriggerV1(name string) TriggerOptionV1 {
- return func(t *eventingv1.Trigger) {
- if name != "" {
- t.Spec.Subscriber = duckv1.Destination{
- Ref: KnativeRefForService(name, t.Namespace),
- }
+ return WithSubscriberDestinationV1(func(t *eventingv1.Trigger) duckv1.Destination {
+ return duckv1.Destination{
+ Ref: KnativeRefForService(name, t.Namespace),
  }
- }
+ })
 }
 
 // WithSubscriberURIForTriggerV1Beta1 returns an option that adds a Subscriber URI for the given v1beta1 Trigger.
 func WithSubscriberURIForTriggerV1Beta1(uri string) TriggerOptionV1Beta1 {
- apisURI, _ := apis.ParseURL(uri)
- return func(t *eventingv1beta1.Trigger) {
- t.Spec.Subscriber = duckv1.Destination{
+ return WithSubscriberDestinationV1Beta1(func(t *eventingv1beta1.Trigger) duckv1.Destination {
+  apisURI, _ := apis.ParseURL(uri)
+ return duckv1.Destination{
  URI: apisURI,
  }
- }
+ })
 }
 
 // WithSubscriberURIForTriggerV1 returns an option that adds a Subscriber URI for the given v1 Trigger.
 func WithSubscriberURIForTriggerV1(uri string) TriggerOptionV1 {
- apisURI, _ := apis.ParseURL(uri)
- return func(t *eventingv1.Trigger) {
- t.Spec.Subscriber = duckv1.Destination{
+ return WithSubscriberDestinationV1(func(t *eventingv1.Trigger) duckv1.Destination {
+  apisURI, _ := apis.ParseURL(uri)
+ return duckv1.Destination{
  URI: apisURI,
  }
+ })
+}
+
+// WithSubscriberDestinationV1Beta1 returns an option that adds a Subscriber for given
+// duckv1.Destination.
+func WithSubscriberDestinationV1Beta1(destFactory func(t *eventingv1beta1.Trigger) duckv1.Destination) TriggerOptionV1Beta1 {
+ return func(t *eventingv1beta1.Trigger) {
+ dest := destFactory(t)
+ if dest.Ref != nil || dest.URI != nil {
+ t.Spec.Subscriber = dest
+ }
+ }
+}
+
+// WithSubscriberDestinationV1 returns an option that adds a Subscriber for given
+// duckv1.Destination.
+func WithSubscriberDestinationV1(destFactory func(t *eventingv1.Trigger) duckv1.Destination) TriggerOptionV1 {
+ return func(t *eventingv1.Trigger) {
+ dest := destFactory(t)
+ if dest.Ref != nil || dest.URI != nil {
+ t.Spec.Subscriber = dest
+ }
  }
 }
 

test/test_images/wathola-forwarder/main_test.go

@@ -29,7 +29,7 @@ import (
 
 func TestForwarderMain(t *testing.T) {
  port := freeport.GetPort()
- config.Instance.LogLevel = zapcore.DebugLevel
+ config.Instance.LogLevel = zapcore.DebugLevel.String()
  config.Instance.Forwarder.Port = port
  go main()
  time.Sleep(time.Millisecond)

test/test_images/wathola-receiver/main_test.go

@@ -30,7 +30,7 @@ import (
 
 func TestReceiverMain(t *testing.T) {
  port := freeport.GetPort()
- config.Instance.LogLevel = zapcore.DebugLevel
+ config.Instance.LogLevel = zapcore.DebugLevel.String()
  config.Instance.Receiver.Port = port
  go main()
  time.Sleep(time.Millisecond)

test/upgrade/README.md

@@ -6,7 +6,8 @@ Running these tests on every commit will ensure that we don’t introduce any
 non-upgradeable changes, so every commit should be releasable.
 
 This is inspired by kubernetes
-[upgrade testing](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-testing/e2e-tests.md#version-skewed-and-upgrade-testing).
+[upgrade testing](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-testing/e2e-tests.md#version-skewed-and-upgrade-testing)
+.
 
 These tests are a pretty big hammer in that they cover more than just version
 changes, but it’s one of the only ways to make sure we don’t accidentally make
@@ -26,15 +27,19 @@ At a high level, we want to do this:
 1. Run any post-install jobs that apply for the release to be.
 1. Test those resources, verify that we didn’t break anything.
 
-To achieve that, we just have three separate build tags:
+To achieve that, we created an upgrade framework (knative.dev/pkg/test/upgrade).
+This framework will enforce running upgrade tests in specific order and supports
+continual verification of system under test. In case of Eventing it is:
 
 1. Install the latest release from GitHub.
-1. Run the `preupgrade` tests in this directory.
-1. Install at HEAD (`ko apply -f config/`).
-1. Run the post-install job. For v0.15 we need to migrate storage versions.
-1. Run the `postupgrade` tests in this directory.
+1. Run the `preupgrade` smoke tests.
+1. Start `continual` tests that will propagate events in the background, while
+ upgrading and downgrading.
+1. Install at HEAD (`ko apply -f config/`) and run the post-install jobs.
+1. Run the `postupgrade` smoke tests.
 1. Install the latest release from GitHub.
-1. Run the `postdowngrade` tests in this directory.
+1. Run the `postdowngrade` smoke tests.
+1. Stop and verify `continual` tests, checking if every event propagated well.
 
 ## Tests
 
@@ -52,42 +57,47 @@ In order to verify that we don't have data-plane unavailability during our
 control-plane outages (when we're upgrading the knative/eventing installation),
 we run a prober test that continually sends events to a service during the
 entire upgrade/downgrade process. When the upgrade completes, we make sure that
-all of those events propagated just once.
-
-To achieve that a [wathola tool](test/upgrade/prober/wathola) was prepared. It
-consists of 4 components: _sender_, _forwarder_, _receiver_, and _fetcher_.
-_Sender_ is the usual Kubernetes deployment that publishes events to the default
-`broker` with given interval. When it terminates (by either `SIGTERM`, or
+all of those events propagated at least once.
+
+To achieve that
+a [wathola tool](https://pkg.go.dev/knative.dev/eventing/test/upgrade/prober/wathola)
+was prepared. It consists of 4 components: _sender_, _forwarder_, _receiver_,
+and _fetcher_. _Sender_ is the usual Kubernetes deployment that publishes events
+to the System Under Tests (SUT). By default, SUT is a default `broker`
+with two triggers for each type of events being sent. _Sender_ will send events
+with given interval. When it terminates (by either `SIGTERM`, or
 `SIGINT`), a `finished` event is generated. _Forwarder_ is a knative serving
 service that scales up from zero to receive the sent events and forward them to
 given target which is the _receiver_ in our case. _Receiver_ is an ordinary
-deployment that collects events from multiple forwarders and has an endpoint
-`/report` that can be polled to get the status of received events. To fetch the
-report from within the cluster _fetcher_ comes in. It's a simple one time job,
-that will fetch the report from _receiver_ and print it on stdout as JSON. That
-enables the test client to download _fetcher_ logs and parse the JSON to get the
-final report.
+deployment that collects events from multiple forwarders and has an
+endpoint `/report` that can be polled to get the status of received events. To
+fetch the report from within the cluster _fetcher_ comes in. It's a simple one
+time job, that will fetch the report from _receiver_ and print it on stdout as
+JSON. That enables the test client to download _fetcher_ logs and parse the JSON
+to get the final report.
 
 Diagram below describe the setup:
 
 ```
  K8s cluster | Test machine
  |
- (deploym.)  (ksvc)  (deploym.)  |
+(deployment) (ksvc) (deployment) |
 +--------+ +-----------+ +----------+ | +------------+
 | | | ++ | | | | |
 | Sender | +-->| Forwarder ||----->+ Receiver | | + TestProber |
 | | | | || | |<---+ | | |
 +---+----+ | +------------| +----------+ | | +------------+
  | | +-----------+ | |
- | | | |
- | | +---------+ |
- | +--+-----+ +---------+ | | |
- +-----> | | +-+ + Fetcher | |
- | Broker | < - > | Trigger | | | | |
- | | | | | +---------+ |
- +--------+ +---------+ | (job) |
- (default) +----------+ |
+ | ```````|````````````````````````````` | |
+ | ` | ` +---------+ |
+ | ` +--+-----+ +---------+ ` | | |
+ +-----> | | +-+ ` | Fetcher | |
+ ` | Broker | < - > | Trigger | | ` | | |
+ ` | | | | | ` +---------+ |
+ ` +--------+ +---------+ | ` (job) |
+ ` (default) +----------+ ` |
+ ` (SUT) `
+ `````````````````````````````````````
 ```
 
 #### Probe test configuration
@@ -96,16 +106,16 @@ Probe test behavior can be influenced from outside without modifying its source
 code. That can be beneficial if one would like to run upgrade tests in different
 context. One such example might be running Eventing upgrade tests in place that
 have Serving and Eventing both installed. In such environment one can set
-environment variable `E2E_UPGRADE_TESTS_SERVING_USE` to enable usage of ksvc
-forwarder (which is disabled by default):
+environment variable `EVENTING_UPGRADE_TESTS_SERVING_USE` to enable usage of
+ksvc forwarder (which is disabled by default):
 
 ```
-$ export E2E_UPGRADE_TESTS_SERVING_USE=true
+$ export EVENTING_UPGRADE_TESTS_SERVING_USE=true
 ```
 
 Any option, apart from namespace, in
 [`knative.dev/eventing/test/upgrade/prober.Config`](https://github.com/knative/eventing/blob/022e281/test/upgrade/prober/prober.go#L52-L63)
-struct can be influenced, by using `E2E_UPGRADE_TESTS_XXXXX` environmental
+struct can be influenced, by using `EVENTING_UPGRADE_TESTS_XXXXX` environmental
 variable prefix (using
 [kelseyhightower/envconfig](https://github.com/kelseyhightower/envconfig#usage)
 usage).

test/upgrade/continual.go

@@ -17,28 +17,12 @@ limitations under the License.
 package upgrade
 
 import (
- "context"
-
- testlib "knative.dev/eventing/test/lib"
  "knative.dev/eventing/test/upgrade/prober"
  pkgupgrade "knative.dev/pkg/test/upgrade"
 )
 
+// ContinualTest will perform a continual validation of Eventing SUT.
 func ContinualTest() pkgupgrade.BackgroundOperation {
- ctx := context.Background()
- var client *testlib.Client
- var probe prober.Prober
- return pkgupgrade.NewBackgroundVerification("EventingContinualTest",
- func(c pkgupgrade.Context) {
- // setup
- client = testlib.Setup(c.T, false)
- config := prober.NewConfig(client.Namespace)
- probe = prober.RunEventProber(ctx, c.Log, client, config)
- },
- func(c pkgupgrade.Context) {
- // verify
- defer testlib.TearDown(client)
- prober.AssertEventProber(ctx, c.T, probe)
- },
- )
+ return prober.NewContinualVerification("EventingContinualTest",
+ prober.ContinualVerificationOptions{})
 }

test/upgrade/prober/config.toml

@@ -1,6 +1,6 @@
-# logLevel = 5 # DEBUG(5)
+# logLevel = 'DEBUG'
 [sender]
-address = '{{- .BrokerURL -}}'
+address = '{{- .Endpoint -}}'
 interval = {{ .Config.Interval.Nanoseconds }}
 [forwarder]
-target = 'http://wathola-receiver.{{- .Config.Namespace -}}.svc.cluster.local'
+target = 'http://wathola-receiver.{{- .Namespace -}}.svc.cluster.local'