Add builds for op-deployer, bugfixes, artifacts downloads (ethereum-o…

…ptimism#12033)

* Add builds for op-deployer, bugfixes, artifacts downloads

Adds Docker builds for op-deployer, makes some bugfixes, and adds support for downloading remote artifacts.

* Apply code scanning fix for arbitrary file access during archive extraction ("zip slip")

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

* fix compile error

* lint

* fix test

* Update from code review, add docker build

* fix versioning

* remove errant dispatch

* update target

---------

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

.circleci/config.yml

@@ -1683,6 +1683,7 @@ workflows:
                 - op-conductor
                 - da-server
                 - op-supervisor
+                - op-deployer
       - cannon-prestate:
           requires:
             - go-mod-download
@@ -1742,6 +1743,7 @@ workflows:
                 - da-server
                 - op-ufm
                 - op-supervisor
+                - op-deployer
           name: <<matrix.docker_name>>-docker-release
           docker_tags: <<pipeline.git.revision>>
           platforms: "linux/amd64,linux/arm64"
@@ -1770,6 +1772,7 @@ workflows:
                 - da-server
                 - op-ufm
                 - op-supervisor
+                - op-deployer
           name: <<matrix.op_component>>-cross-platform
           requires:
             - op-node-docker-release
@@ -1781,6 +1784,7 @@ workflows:
             - da-server-docker-release
             - op-ufm-docker-release
             - op-supervisor-docker-release
+            - op-deployer-docker-release
       # Standard (xlarge) AMD-only docker images go here
       - docker-build:
           matrix:

docker-bake.hcl

@@ -69,6 +69,9 @@ variable "OP_CONDUCTOR_VERSION" {
   default = "${GIT_VERSION}"
 }
 
+variable "OP_DEPLOYER_VERSION" {
+  default = "${GIT_VERSION}"
+}
 
 target "op-node" {
   dockerfile = "ops/docker/op-stack-go/Dockerfile"
@@ -236,3 +239,16 @@ target "contracts-bedrock" {
   platforms = ["linux/amd64"]
   tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/contracts-bedrock:${tag}"]
 }
+
+target "op-deployer" {
+  dockerfile = "ops/docker/op-stack-go/Dockerfile"
+  context = "."
+  args = {
+    GIT_COMMIT = "${GIT_COMMIT}"
+    GIT_DATE = "${GIT_DATE}"
+    OP_DEPLOYER_VERSION = "${OP_DEPLOYER_VERSION}"
+  }
+  target = "op-deployer-target"
+  platforms = split(",", PLATFORMS)
+  tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-deployer:${tag}"]
+}

op-chain-ops/Makefile

@@ -1,3 +1,26 @@
+GITCOMMIT ?= $(shell git rev-parse HEAD)
+GITDATE ?= $(shell git show -s --format='%ct')
+
+# Find the github tag that points to this commit. If none are found, set the version string to "untagged"
+# Prioritizes release tag, if one exists, over tags suffixed with "-rc"
+VERSION ?= $(shell tags=$$(git tag --points-at $(GITCOMMIT) | grep '^op-deployer/' | sed 's/op-deployer\///' | sort -V); \
+             preferred_tag=$$(echo "$$tags" | grep -v -- '-rc' | tail -n 1); \
+             if [ -z "$$preferred_tag" ]; then \
+                 if [ -z "$$tags" ]; then \
+                     echo "untagged"; \
+                 else \
+                     echo "$$tags" | tail -n 1; \
+                 fi \
+             else \
+                 echo $$preferred_tag; \
+             fi)
+
+LDFLAGSSTRING +=-X main.GitCommit=$(GITCOMMIT)
+LDFLAGSSTRING +=-X main.GitDate=$(GITDATE)
+LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/op-chain-ops/deployer/version.Version=$(VERSION)
+LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/op-chain-ops/deployer/version.Meta=$(VERSION_META)
+LDFLAGS := -ldflags "$(LDFLAGSSTRING)"
+
 # Use the old Apple linker to workaround broken xcode - https://github.com/golang/go/issues/65169
 ifeq ($(shell uname),Darwin)
 	FUZZLDFLAGS := -ldflags=-extldflags=-Wl,-ld_classic
@@ -13,7 +36,7 @@ test:
 	go test ./...
 
 op-deployer:
-	go build -o ./bin/op-deployer ./cmd/op-deployer/main.go
+	GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) CGO_ENABLED=0 go build -v $(LDFLAGS) -o ./bin/op-deployer ./cmd/op-deployer/main.go
 
 fuzz:
 	go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime 10s -fuzz=FuzzEncodeDecodeWithdrawal ./crossdomain

op-chain-ops/cmd/op-deployer/main.go

@@ -4,15 +4,27 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/version"
+	opservice "github.com/ethereum-optimism/optimism/op-service"
+
 	"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/inspect"
 
 	"github.com/ethereum-optimism/optimism/op-chain-ops/deployer"
 	"github.com/ethereum-optimism/optimism/op-service/cliapp"
 	"github.com/urfave/cli/v2"
 )
 
+var (
+	GitCommit = ""
+	GitDate   = ""
+)
+
+// VersionWithMeta holds the textual version string including the metadata.
+var VersionWithMeta = opservice.FormatVersion(version.Version, GitCommit, GitDate, version.Meta)
+
 func main() {
 	app := cli.NewApp()
+	app.Version = VersionWithMeta
 	app.Name = "op-deployer"
 	app.Usage = "Tool to configure and deploy OP Chains."
 	app.Flags = cliapp.ProtectFlags(deployer.GlobalFlags)

op-chain-ops/deployer/apply.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/ethereum-optimism/optimism/op-chain-ops/foundry"
+
 	"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/state"
 
 	"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/pipeline"
@@ -128,7 +130,7 @@ func Apply(ctx context.Context, cfg ApplyConfig) error {
 
 type pipelineStage struct {
 	name  string
-	stage pipeline.Stage
+	apply pipeline.Stage
 }
 
 func ApplyPipeline(
@@ -137,29 +139,44 @@ func ApplyPipeline(
 	intent *state.Intent,
 	st *state.State,
 ) error {
+	progressor := func(curr, total int64) {
+		env.Logger.Info("artifacts download progress", "current", curr, "total", total)
+	}
+
+	artifactsFS, cleanup, err := pipeline.DownloadArtifacts(ctx, intent.ContractArtifactsURL, progressor)
+	if err != nil {
+		return fmt.Errorf("failed to download artifacts: %w", err)
+	}
+	defer func() {
+		if err := cleanup(); err != nil {
+			env.Logger.Warn("failed to clean up artifacts", "err", err)
+		}
+	}()
+
 	pline := []pipelineStage{
 		{"init", pipeline.Init},
 		{"deploy-superchain", pipeline.DeploySuperchain},
 		{"deploy-implementations", pipeline.DeployImplementations},
 	}
 
 	for _, chain := range intent.Chains {
+		chainID := chain.ID
 		pline = append(pline, pipelineStage{
-			fmt.Sprintf("deploy-opchain-%s", chain.ID.Hex()),
-			func(ctx context.Context, env *pipeline.Env, intent *state.Intent, st *state.State) error {
-				return pipeline.DeployOPChain(ctx, env, intent, st, chain.ID)
+			fmt.Sprintf("deploy-opchain-%s", chainID.Hex()),
+			func(ctx context.Context, env *pipeline.Env, artifactsFS foundry.StatDirFs, intent *state.Intent, st *state.State) error {
+				return pipeline.DeployOPChain(ctx, env, artifactsFS, intent, st, chainID)
 			},
 		}, pipelineStage{
-			fmt.Sprintf("generate-l2-genesis-%s", chain.ID.Hex()),
-			func(ctx context.Context, env *pipeline.Env, intent *state.Intent, st *state.State) error {
-				return pipeline.GenerateL2Genesis(ctx, env, intent, st, chain.ID)
+			fmt.Sprintf("generate-l2-genesis-%s", chainID.Hex()),
+			func(ctx context.Context, env *pipeline.Env, artifactsFS foundry.StatDirFs, intent *state.Intent, st *state.State) error {
+				return pipeline.GenerateL2Genesis(ctx, env, artifactsFS, intent, st, chainID)
 			},
 		})
 	}
 
 	for _, stage := range pline {
-		if err := stage.stage(ctx, env, intent, st); err != nil {
-			return fmt.Errorf("error in pipeline stage: %w", err)
+		if err := stage.apply(ctx, env, artifactsFS, intent, st); err != nil {
+			return fmt.Errorf("error in pipeline stage apply: %w", err)
 		}
 	}
 

op-chain-ops/deployer/flags.go

@@ -12,9 +12,9 @@ const (
 	EnvVarPrefix       = "DEPLOYER"
 	L1RPCURLFlagName   = "l1-rpc-url"
 	L1ChainIDFlagName  = "l1-chain-id"
+	L2ChainIDsFlagName = "l2-chain-ids"
 	WorkdirFlagName    = "workdir"
 	OutdirFlagName     = "outdir"
-	DevFlagName        = "dev"
 	PrivateKeyFlagName = "private-key"
 )
 
@@ -33,6 +33,11 @@ var (
 		EnvVars: prefixEnvVar("L1_CHAIN_ID"),
 		Value:   900,
 	}
+	L2ChainIDsFlag = &cli.StringFlag{
+		Name:    L2ChainIDsFlagName,
+		Usage:   "Comma-separated list of L2 chain IDs to deploy.",
+		EnvVars: prefixEnvVar("L2_CHAIN_IDS"),
+	}
 	WorkdirFlag = &cli.StringFlag{
 		Name:    WorkdirFlagName,
 		Usage:   "Directory storing intent and stage. Defaults to the current directory.",
@@ -42,12 +47,6 @@ var (
 			OutdirFlagName,
 		},
 	}
-	DevFlag = &cli.BoolFlag{
-		Name: DevFlagName,
-		Usage: "Use development mode. This will use the development mnemonic to own the chain" +
-			" and fund dev accounts.",
-		EnvVars: prefixEnvVar("DEV"),
-	}
 
 	PrivateKeyFlag = &cli.StringFlag{
 		Name:    PrivateKeyFlagName,
@@ -60,8 +59,8 @@ var GlobalFlags = append([]cli.Flag{}, oplog.CLIFlags(EnvVarPrefix)...)
 
 var InitFlags = []cli.Flag{
 	L1ChainIDFlag,
+	L2ChainIDsFlag,
 	WorkdirFlag,
-	DevFlag,
 }
 
 var ApplyFlags = []cli.Flag{

op-chain-ops/deployer/init.go

@@ -3,19 +3,20 @@ package deployer
 import (
 	"fmt"
 	"path"
+	"strings"
+
+	op_service "github.com/ethereum-optimism/optimism/op-service"
 
 	"github.com/ethereum-optimism/optimism/op-chain-ops/deployer/state"
 	"github.com/ethereum-optimism/optimism/op-chain-ops/devkeys"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/urfave/cli/v2"
 )
 
-const devMnemonic = "test test test test test test test test test test test junk"
-
 type InitConfig struct {
-	L1ChainID uint64
-	Outdir    string
-	Dev       bool
+	L1ChainID  uint64
+	Outdir     string
+	L2ChainIDs []common.Hash
 }
 
 func (c *InitConfig) Check() error {
@@ -27,19 +28,33 @@ func (c *InitConfig) Check() error {
 		return fmt.Errorf("outdir must be specified")
 	}
 
+	if len(c.L2ChainIDs) == 0 {
+		return fmt.Errorf("must specify at least one L2 chain ID")
+	}
+
 	return nil
 }
 
 func InitCLI() func(ctx *cli.Context) error {
 	return func(ctx *cli.Context) error {
 		l1ChainID := ctx.Uint64(L1ChainIDFlagName)
 		outdir := ctx.String(OutdirFlagName)
-		dev := ctx.Bool(DevFlagName)
+
+		l2ChainIDsRaw := ctx.String(L2ChainIDsFlagName)
+		l2ChainIDsStr := strings.Split(l2ChainIDsRaw, ",")
+		l2ChainIDs := make([]common.Hash, 0, len(l2ChainIDsStr))
+		for _, idStr := range l2ChainIDsStr {
+			id, err := op_service.Parse256BitChainID(idStr)
+			if err != nil {
+				return fmt.Errorf("invalid chain ID: %w", err)
+			}
+			l2ChainIDs = append(l2ChainIDs, id)
+		}
 
 		return Init(InitConfig{
-			L1ChainID: l1ChainID,
-			Outdir:    outdir,
-			Dev:       dev,
+			L1ChainID:  l1ChainID,
+			Outdir:     outdir,
+			L2ChainIDs: l2ChainIDs,
 		})
 	}
 }
@@ -52,30 +67,44 @@ func Init(cfg InitConfig) error {
 	intent := &state.Intent{
 		L1ChainID:       cfg.L1ChainID,
 		UseFaultProofs:  true,
-		FundDevAccounts: cfg.Dev,
+		FundDevAccounts: true,
 	}
 
 	l1ChainIDBig := intent.L1ChainIDBig()
 
-	if cfg.Dev {
-		dk, err := devkeys.NewMnemonicDevKeys(devMnemonic)
+	dk, err := devkeys.NewMnemonicDevKeys(devkeys.TestMnemonic)
+	if err != nil {
+		return fmt.Errorf("failed to create dev keys: %w", err)
+	}
+
+	addrFor := func(key devkeys.Key) common.Address {
+		// The error below should never happen, so panic if it does.
+		addr, err := dk.Address(key)
 		if err != nil {
-			return fmt.Errorf("failed to create dev keys: %w", err)
+			panic(err)
 		}
+		return addr
+	}
+	intent.SuperchainRoles = state.SuperchainRoles{
+		ProxyAdminOwner:       addrFor(devkeys.L1ProxyAdminOwnerRole.Key(l1ChainIDBig)),
+		ProtocolVersionsOwner: addrFor(devkeys.SuperchainProtocolVersionsOwner.Key(l1ChainIDBig)),
+		Guardian:              addrFor(devkeys.SuperchainConfigGuardianKey.Key(l1ChainIDBig)),
+	}
 
-		addrFor := func(key devkeys.Key) common.Address {
-			// The error below should never happen, so panic if it does.
-			addr, err := dk.Address(key)
-			if err != nil {
-				panic(err)
-			}
-			return addr
-		}
-		intent.SuperchainRoles = state.SuperchainRoles{
-			ProxyAdminOwner:       addrFor(devkeys.L1ProxyAdminOwnerRole.Key(l1ChainIDBig)),
-			ProtocolVersionsOwner: addrFor(devkeys.SuperchainDeployerKey.Key(l1ChainIDBig)),
-			Guardian:              addrFor(devkeys.SuperchainConfigGuardianKey.Key(l1ChainIDBig)),
-		}
+	for _, l2ChainID := range cfg.L2ChainIDs {
+		l2ChainIDBig := l2ChainID.Big()
+		intent.Chains = append(intent.Chains, &state.ChainIntent{
+			ID: l2ChainID,
+			Roles: state.ChainRoles{
+				ProxyAdminOwner:      addrFor(devkeys.L2ProxyAdminOwnerRole.Key(l2ChainIDBig)),
+				SystemConfigOwner:    addrFor(devkeys.SystemConfigOwner.Key(l2ChainIDBig)),
+				GovernanceTokenOwner: addrFor(devkeys.L2ProxyAdminOwnerRole.Key(l2ChainIDBig)),
+				UnsafeBlockSigner:    addrFor(devkeys.SequencerP2PRole.Key(l2ChainIDBig)),
+				Batcher:              addrFor(devkeys.BatcherRole.Key(l2ChainIDBig)),
+				Proposer:             addrFor(devkeys.ProposerRole.Key(l2ChainIDBig)),
+				Challenger:           addrFor(devkeys.ChallengerRole.Key(l2ChainIDBig)),
+			},
+		})
 	}
 
 	st := &state.State{