op-supervisor,op-node: introduce follow/managed interop mode (#13285)

* op-supervisor,op-node: introduce follow/managed interop mode v2

* interop: test workaround, serve supervisor while using old deriver

* interop: fix API setup/usage, fix action test setup

* op-node: fix interop server endpoint getter

* interop: minor fixes, update TODO references

interop-devnet/docker-compose.yml

@@ -89,11 +89,10 @@ services:
       op-supervisor
       --datadir="/db"
       --dependency-set="/depset.json"
-      --l2-rpcs=""
       --rpc.addr="0.0.0.0"
       --rpc.port=8545
       --rpc.enable-admin
-      --l2-rpcs="ws://l2-a:8546,ws://l2-b:8546"
+      --l2-consensus.nodes="http://op-node-a:9645,http://op-node-b:9645"
     environment:
       OP_SUPERVISOR_METRICS_ENABLED: "true"
 
@@ -157,7 +156,9 @@ services:
       --l1.http-poll-interval=6s
       --l2=http://l2-a:8551
       --l2.jwt-secret=/config/jwt-secret.txt
-      --supervisor=http://op-supervisor:8545
+      --interop.supervisor=http://op-supervisor:8545
+      --interop.rpc.addr=0.0.0.0
+      --interop.rpc.port=9645
       --sequencer.enabled
       --sequencer.l1-confs=0
       --verifier.l1-confs=0
@@ -180,6 +181,7 @@ services:
       - "9103:9003"
       - "7100:7300"
       - "6160:6060"
+      - "9645:9645"
     volumes:
       - "safedb_a_data:/db"
       - "${PWD}/../ops-bedrock/test-jwt-secret.txt:/config/jwt-secret.txt"
@@ -208,7 +210,9 @@ services:
       --l1.http-poll-interval=6s
       --l2=http://l2-b:8551
       --l2.jwt-secret=/config/jwt-secret.txt
-      --supervisor=http://op-supervisor:8545
+      --interop.supervisor=http://op-supervisor:8545
+      --interop.rpc.addr=0.0.0.0
+      --interop.rpc.port=9645
       --sequencer.enabled
       --sequencer.l1-confs=0
       --verifier.l1-confs=0
@@ -231,6 +235,7 @@ services:
       - "9203:9003"
       - "7200:7300"
       - "6260:6060"
+      - "9645:9645"
     volumes:
       - "safedb_b_data:/db"
       - "${PWD}/../ops-bedrock/test-jwt-secret.txt:/config/jwt-secret.txt"

op-e2e/actions/helpers/l2_verifier.go

@@ -5,11 +5,13 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/big"
 
 	"github.com/stretchr/testify/require"
 	"golang.org/x/time/rate"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
 	gnode "github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -31,6 +33,7 @@ import (
 	"github.com/ethereum-optimism/optimism/op-service/safego"
 	"github.com/ethereum-optimism/optimism/op-service/sources"
 	"github.com/ethereum-optimism/optimism/op-service/testutils"
+	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/backend/syncsrc"
 )
 
 // L2Verifier is an actor that functions like a rollup node,
@@ -65,6 +68,8 @@ type L2Verifier struct {
 
 	rpc *rpc.Server
 
+	interopRPC *rpc.Server
+
 	failRPC func(call []rpc.BatchElem) error // mock error
 
 	// The L2Verifier actor is embedded in the L2Sequencer actor,
@@ -79,6 +84,10 @@ type L2API interface {
 	// GetProof returns a proof of the account, it may return a nil result without error if the address was not found.
 	GetProof(ctx context.Context, address common.Address, storage []common.Hash, blockTag string) (*eth.AccountResult, error)
 	OutputV0AtBlock(ctx context.Context, blockHash common.Hash) (*eth.OutputV0, error)
+
+	FetchReceipts(ctx context.Context, blockHash common.Hash) (eth.BlockInfo, types.Receipts, error)
+	BlockRefByNumber(ctx context.Context, num uint64) (eth.BlockRef, error)
+	ChainID(ctx context.Context) (*big.Int, error)
 }
 
 type safeDB interface {
@@ -181,6 +190,13 @@ func NewL2Verifier(t Testing, log log.Logger, l1 derive.L1Fetcher,
 
 	t.Cleanup(rollupNode.rpc.Stop)
 
+	if cfg.InteropTime != nil {
+		rollupNode.interopRPC = rpc.NewServer()
+		api := &interop.TemporaryInteropAPI{Eng: eng}
+		require.NoError(t, rollupNode.interopRPC.RegisterName("interop", api))
+		t.Cleanup(rollupNode.interopRPC.Stop)
+	}
+
 	// setup RPC server for rollup node, hooked to the actor as backend
 	m := &testutils.TestRPCMetrics{}
 	backend := &l2VerifierBackend{verifier: rollupNode}
@@ -203,6 +219,13 @@ func NewL2Verifier(t Testing, log log.Logger, l1 derive.L1Fetcher,
 	return rollupNode
 }
 
+func (v *L2Verifier) InteropSyncSource(t Testing) syncsrc.SyncSource {
+	require.NotNil(t, v.interopRPC, "interop rpc must be running")
+	cl := rpc.DialInProc(v.interopRPC)
+	bCl := client.NewBaseRPCClient(cl)
+	return syncsrc.NewRPCSyncSource("action-tests-l2-verifier", bCl)
+}
+
 type l2VerifierBackend struct {
 	verifier *L2Verifier
 }

op-e2e/actions/interop/interop.go

@@ -27,6 +27,7 @@ import (
 	"github.com/ethereum-optimism/optimism/op-supervisor/metrics"
 	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/backend"
 	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/backend/depset"
+	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/backend/syncsrc"
 	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/frontend"
 	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types"
 )
@@ -109,8 +110,10 @@ func (is *InteropSetup) CreateActors() *InteropActors {
 	chainA := createL2Services(is.T, is.Log, l1Miner, is.Keys, is.Out.L2s["900200"], supervisorAPI)
 	chainB := createL2Services(is.T, is.Log, l1Miner, is.Keys, is.Out.L2s["900201"], supervisorAPI)
 	// Hook up L2 RPCs to supervisor, to fetch event data from
-	require.NoError(is.T, supervisorAPI.AddL2RPC(is.T.Ctx(), chainA.SequencerEngine.HTTPEndpoint()))
-	require.NoError(is.T, supervisorAPI.AddL2RPC(is.T.Ctx(), chainB.SequencerEngine.HTTPEndpoint()))
+	srcA := chainA.Sequencer.InteropSyncSource(is.T)
+	srcB := chainB.Sequencer.InteropSyncSource(is.T)
+	require.NoError(is.T, supervisorAPI.backend.AttachSyncSource(is.T.Ctx(), srcA))
+	require.NoError(is.T, supervisorAPI.backend.AttachSyncSource(is.T.Ctx(), srcB))
 	return &InteropActors{
 		L1Miner:    l1Miner,
 		Supervisor: supervisorAPI,
@@ -163,6 +166,7 @@ func NewSupervisor(t helpers.Testing, logger log.Logger, depSet depset.Dependenc
 		DependencySetSource:   depSet,
 		SynchronousProcessors: true,
 		Datadir:               supervisorDataDir,
+		SyncSources:           &syncsrc.CLISyncSources{}, // sources are added dynamically afterwards
 	}
 	b, err := backend.NewSupervisorBackend(t.Ctx(),
 		logger.New("role", "supervisor"), metrics.NoopMetrics, svCfg)

op-e2e/e2eutils/opnode/opnode.go

@@ -11,12 +11,17 @@ import (
 	"github.com/ethereum-optimism/optimism/op-node/p2p"
 	"github.com/ethereum-optimism/optimism/op-service/cliapp"
 	"github.com/ethereum-optimism/optimism/op-service/endpoint"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
 )
 
 type Opnode struct {
 	node *rollupNode.OpNode
 }
 
+func (o *Opnode) InteropRPC() (endpoint string, jwtSecret eth.Bytes32) {
+	return o.node.InteropRPC()
+}
+
 func (o *Opnode) UserRPC() endpoint.RPC {
 	return endpoint.HttpURL(o.node.HTTPEndpoint())
 }

op-e2e/interop/supersystem.go

@@ -11,26 +11,19 @@ import (
 	"testing"
 	"time"
 
-	"github.com/ethereum/go-ethereum/eth/ethconfig"
-	gn "github.com/ethereum/go-ethereum/node"
-
 	"github.com/stretchr/testify/require"
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/log"
+	gn "github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"
 
-	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/interop/contracts/bindings/emit"
-	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/interop/contracts/bindings/inbox"
-	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/interop/contracts/bindings/systemconfig"
-	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait"
-	"github.com/ethereum-optimism/optimism/op-service/predeploys"
-
 	bss "github.com/ethereum-optimism/optimism/op-batcher/batcher"
 	batcherFlags "github.com/ethereum-optimism/optimism/op-batcher/flags"
 	"github.com/ethereum-optimism/optimism/op-chain-ops/devkeys"
@@ -39,14 +32,19 @@ import (
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/fakebeacon"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/geth"
+	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/interop/contracts/bindings/emit"
+	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/interop/contracts/bindings/inbox"
+	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/interop/contracts/bindings/systemconfig"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/opnode"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/services"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/setuputils"
+	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait"
 	"github.com/ethereum-optimism/optimism/op-e2e/system/helpers"
 	"github.com/ethereum-optimism/optimism/op-node/node"
 	"github.com/ethereum-optimism/optimism/op-node/p2p"
 	"github.com/ethereum-optimism/optimism/op-node/rollup/derive"
 	"github.com/ethereum-optimism/optimism/op-node/rollup/driver"
+	"github.com/ethereum-optimism/optimism/op-node/rollup/interop"
 	"github.com/ethereum-optimism/optimism/op-node/rollup/sync"
 	l2os "github.com/ethereum-optimism/optimism/op-proposer/proposer"
 	"github.com/ethereum-optimism/optimism/op-service/client"
@@ -56,12 +54,14 @@ import (
 	oplog "github.com/ethereum-optimism/optimism/op-service/log"
 	"github.com/ethereum-optimism/optimism/op-service/metrics"
 	"github.com/ethereum-optimism/optimism/op-service/oppprof"
+	"github.com/ethereum-optimism/optimism/op-service/predeploys"
 	oprpc "github.com/ethereum-optimism/optimism/op-service/rpc"
 	"github.com/ethereum-optimism/optimism/op-service/sources"
 	"github.com/ethereum-optimism/optimism/op-service/testlog"
 	supervisorConfig "github.com/ethereum-optimism/optimism/op-supervisor/config"
 	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor"
 	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/backend/depset"
+	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/backend/syncsrc"
 	supervisortypes "github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types"
 )
 
@@ -317,8 +317,11 @@ func (s *interopE2ESystem) newNodeForL2(
 			ListenPort:  0,
 			EnableAdmin: true,
 		},
-		Supervisor: &node.SupervisorEndpointConfig{
-			SupervisorAddr: s.supervisor.RPC(),
+		InteropConfig: &interop.Config{
+			SupervisorAddr:   s.supervisor.RPC(),
+			RPCAddr:          "127.0.0.1",
+			RPCPort:          0,
+			RPCJwtSecretPath: "",
 		},
 		P2P:                         nil, // disabled P2P setup for now
 		L1EpochPollInterval:         time.Second * 2,
@@ -479,9 +482,9 @@ func (s *interopE2ESystem) prepareSupervisor() *supervisor.SupervisorService {
 			ListenPort:  0,
 			EnableAdmin: true,
 		},
-		L2RPCs:  []string{},
-		L1RPC:   s.l1.UserRPC().RPC(),
-		Datadir: path.Join(s.t.TempDir(), "supervisor"),
+		SyncSources: &syncsrc.CLISyncSources{}, // no sync-sources
+		L1RPC:       s.l1.UserRPC().RPC(),
+		Datadir:     path.Join(s.t.TempDir(), "supervisor"),
 	}
 	depSet := make(map[supervisortypes.ChainID]*depset.StaticConfigDependency)
 
@@ -549,7 +552,8 @@ func (s *interopE2ESystem) prepare(t *testing.T, w worldResourcePaths) {
 	// add the L2 RPCs to the supervisor now that the L2s are created
 	ctx := context.Background()
 	for _, l2 := range s.l2s {
-		err := s.SupervisorClient().AddL2RPC(ctx, l2.l2Geth.UserRPC().RPC())
+		rpcEndpoint, secret := l2.opNode.InteropRPC()
+		err := s.SupervisorClient().AddL2RPC(ctx, rpcEndpoint, secret)
 		require.NoError(s.t, err, "failed to add L2 RPC to supervisor")
 	}
 

op-node/flags/flags.go

@@ -28,6 +28,7 @@ const (
 	P2PCategory        = "5. PEER-TO-PEER"
 	AltDACategory      = "6. ALT-DA (EXPERIMENTAL)"
 	MiscCategory       = "7. MISC"
+	InteropCategory    = "8. INTEROP (SUPER EXPERIMENTAL)"
 )
 
 func init() {
@@ -74,13 +75,6 @@ var (
 		Category: RollupCategory,
 	}
 	/* Optional Flags */
-	SupervisorAddr = &cli.StringFlag{
-		Name: "supervisor",
-		Usage: "RPC address of interop supervisor service for cross-chain safety verification." +
-			"Applies only to Interop-enabled networks.",
-		Hidden:  true, // hidden for now during early testing.
-		EnvVars: prefixEnvVars("SUPERVISOR"),
-	}
 	BeaconHeader = &cli.StringFlag{
 		Name:     "l1.beacon-header",
 		Usage:    "Optional HTTP header to add to all requests to the L1 Beacon endpoint. Format: 'X-Key: Value'",
@@ -372,6 +366,40 @@ var (
 		Value:    time.Second * 1,
 		Category: SequencerCategory,
 	}
+	/* Interop flags, experimental. */
+	InteropSupervisor = &cli.StringFlag{
+		Name: "interop.supervisor",
+		Usage: "Interop standard-mode: RPC address of interop supervisor to use for cross-chain safety verification." +
+			"Applies only to Interop-enabled networks.",
+		EnvVars:  prefixEnvVars("INTEROP_SUPERVISOR"),
+		Category: InteropCategory,
+	}
+	InteropRPCAddr = &cli.StringFlag{
+		Name: "interop.rpc.addr",
+		Usage: "Interop Websocket-only RPC listening address, to serve supervisor syncing." +
+			"Applies only to Interop-enabled networks. Optional, alternative to follow-mode.",
+		EnvVars:  prefixEnvVars("INTEROP_RPC_ADDR"),
+		Value:    "127.0.0.1",
+		Category: InteropCategory,
+	}
+	InteropRPCPort = &cli.IntFlag{
+		Name: "interop.rpc.port",
+		Usage: "Interop RPC listening port, to serve supervisor syncing." +
+			"Applies only to Interop-enabled networks.",
+		EnvVars:  prefixEnvVars("INTEROP_RPC_PORT"),
+		Value:    9645, // Note: op-service/rpc/cli.go uses 8545 as the default.
+		Category: InteropCategory,
+	}
+	InteropJWTSecret = &cli.StringFlag{
+		Name: "interop.jwt-secret",
+		Usage: "Interop RPC server authentication. Path to JWT secret key. Keys are 32 bytes, hex encoded in a file. " +
+			"A new key will be generated if the file is empty. " +
+			"Applies only to Interop-enabled networks.",
+		EnvVars:     prefixEnvVars("INTEROP_JWT_SECRET"),
+		Value:       "",
+		Destination: new(string),
+		Category:    InteropCategory,
+	}
 )
 
 var requiredFlags = []cli.Flag{
@@ -381,7 +409,6 @@ var requiredFlags = []cli.Flag{
 }
 
 var optionalFlags = []cli.Flag{
-	SupervisorAddr,
 	BeaconAddr,
 	BeaconHeader,
 	BeaconFallbackAddrs,
@@ -419,6 +446,10 @@ var optionalFlags = []cli.Flag{
 	ConductorRpcTimeoutFlag,
 	SafeDBPath,
 	L2EngineKind,
+	InteropSupervisor,
+	InteropRPCAddr,
+	InteropRPCPort,
+	InteropJWTSecret,
 }
 
 var DeprecatedFlags = []cli.Flag{

op-node/node/client.go

@@ -230,29 +230,3 @@ func parseHTTPHeader(headerStr string) (http.Header, error) {
 	h.Add(s[0], s[1])
 	return h, nil
 }
-
-type SupervisorEndpointSetup interface {
-	SupervisorClient(ctx context.Context, log log.Logger) (*sources.SupervisorClient, error)
-	Check() error
-}
-
-type SupervisorEndpointConfig struct {
-	SupervisorAddr string
-}
-
-var _ SupervisorEndpointSetup = (*SupervisorEndpointConfig)(nil)
-
-func (cfg *SupervisorEndpointConfig) Check() error {
-	if cfg.SupervisorAddr == "" {
-		return errors.New("supervisor RPC address is not set")
-	}
-	return nil
-}
-
-func (cfg *SupervisorEndpointConfig) SupervisorClient(ctx context.Context, log log.Logger) (*sources.SupervisorClient, error) {
-	cl, err := client.NewRPC(ctx, log, cfg.SupervisorAddr, client.WithLazyDial())
-	if err != nil {
-		return nil, fmt.Errorf("failed to create supervisor RPC: %w", err)
-	}
-	return sources.NewSupervisorClient(cl), nil
-}