diff --git a/.changelog/10302.txt b/.changelog/10302.txt
new file mode 100644
index 000000000000..e87d168986b5
--- /dev/null
+++ b/.changelog/10302.txt
@@ -0,0 +1,4 @@
+```release-note:improvement
+connect: Avoid adding original_dst listener filter when it won't be used.
+```
+
diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go
index 17dc11a4df06..c31f0fd6c59c 100644
--- a/agent/xds/listeners.go
+++ b/agent/xds/listeners.go
@@ -78,12 +78,6 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
 
 		outboundListener = makePortListener(OutboundListenerName, "127.0.0.1", port, envoy_core_v3.TrafficDirection_OUTBOUND)
 		outboundListener.FilterChains = make([]*envoy_listener_v3.FilterChain, 0)
-		outboundListener.ListenerFilters = []*envoy_listener_v3.ListenerFilter{
-			{
-				// TODO (freddy): Hard-coded until we upgrade the go-control-plane library
-				Name: "envoy.filters.listener.original_dst",
-			},
-		}
 	}
 
 	var hasFilterChains bool
@@ -207,6 +201,13 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
 		if cfgSnap.ConnectProxy.MeshConfig == nil ||
 			!cfgSnap.ConnectProxy.MeshConfig.TransparentProxy.CatalogDestinationsOnly {
 
+			outboundListener.ListenerFilters = []*envoy_listener_v3.ListenerFilter{
+				{
+					// TODO (freddy): Hard-coded until we upgrade the go-control-plane library
+					Name: "envoy.filters.listener.original_dst",
+				},
+			}
+
 			filterChain, err := s.makeUpstreamFilterChainForDiscoveryChain(
 				"passthrough",
 				OriginalDestinationClusterName,
diff --git a/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.envoy-1-18-x.golden b/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.envoy-1-18-x.golden
index a3c52dc4f3ad..4c444e0444cc 100644
--- a/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.envoy-1-18-x.golden
+++ b/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.envoy-1-18-x.golden
@@ -57,11 +57,6 @@
           ]
         }
       ],
-      "listenerFilters": [
-        {
-          "name": "envoy.filters.listener.original_dst"
-        }
-      ],
       "trafficDirection": "OUTBOUND"
     },
     {
diff --git a/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.v2compat.envoy-1-16-x.golden b/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.v2compat.envoy-1-16-x.golden
index db881a384a35..1d3f29225d2d 100644
--- a/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.v2compat.envoy-1-16-x.golden
+++ b/agent/xds/testdata/listeners/transparent-proxy-catalog-destinations-only.v2compat.envoy-1-16-x.golden
@@ -57,11 +57,6 @@
           ]
         }
       ],
-      "listenerFilters": [
-        {
-          "name": "envoy.filters.listener.original_dst"
-        }
-      ],
       "trafficDirection": "OUTBOUND"
     },
     {