Avoid retrying rate limited events

runtime/events/recorder.go

@@ -17,8 +17,10 @@ limitations under the License.
 package events
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"net/url"
 	"os"
 	"time"
@@ -49,6 +51,7 @@ func NewRecorder(webhook, reportingController string) (*Recorder, error) {
 
 	httpClient := retryablehttp.NewClient()
 	httpClient.HTTPClient.Timeout = 5 * time.Second
+	httpClient.CheckRetry = retryablehttp.ErrorPropagatedRetryPolicy
 	httpClient.Logger = nil
 
 	return &Recorder{
@@ -120,6 +123,12 @@ func (r *Recorder) Eventf(
 		return fmt.Errorf("failed to marshal object into json, error: %w", err)
 	}
 
+	// avoid retrying rate limited requests
+	if res, _ := r.Client.HTTPClient.Post(r.Webhook, "application/json", bytes.NewReader(body)); res != nil &&
+		(res.StatusCode == http.StatusTooManyRequests || res.StatusCode == http.StatusAccepted) {
+		return nil
+	}
+
 	if _, err := r.Client.Post(r.Webhook, "application/json", body); err != nil {
 		return err
 	}

runtime/events/recorder_test.go

@@ -18,7 +18,6 @@ package events
 
 import (
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -86,5 +85,32 @@ func TestEventRecorder_Eventf_Retry(t *testing.T) {
 	}
 
 	err = eventRecorder.EventErrorf(obj, nil, "sync", "sync %s", obj.Name)
-	require.EqualError(t, err, fmt.Sprintf("POST %s giving up after 3 attempt(s)", ts.URL))
+	require.Error(t, err)
+}
+
+func TestEventRecorder_Eventf_RateLimited(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+
+		var payload Event
+		err = json.Unmarshal(b, &payload)
+		require.NoError(t, err)
+
+		w.WriteHeader(http.StatusTooManyRequests)
+	}))
+	defer ts.Close()
+
+	eventRecorder, err := NewRecorder(ts.URL, "test-controller")
+	require.NoError(t, err)
+	eventRecorder.Client.RetryMax = 2
+
+	obj := corev1.ObjectReference{
+		Kind:      "GitRepository",
+		Namespace: "gitops-system",
+		Name:      "webapp",
+	}
+
+	err = eventRecorder.EventInfof(obj, nil, "sync", "sync %s", obj.Name)
+	require.NoError(t, err)
 }