feat: Curio market migration command

.github/image/Dockerfile

@@ -18,7 +18,7 @@ RUN git submodule update --init
 RUN go mod download
 
 # Download Go dependencies for Curio
-WORKDIR /app/boost
+WORKDIR /app/curio
 RUN git submodule update --init
 RUN go mod download
 

.gitignore

@@ -8,6 +8,7 @@
 /booster-bitswap
 /docgen-md
 /docgen-openrpc
+/migrate-curio
 extern/filecoin-ffi/rust/target
 extern/boostd-data/boostd-data
 **/*.a

Makefile

@@ -133,6 +133,11 @@ boostci: $(BUILD_DEPS)
 	$(GOCC) build $(GOFLAGS) -o boostci ./cmd/boostci
 .PHONY: boostci
 
+migrate-curio: $(BUILD_DEPS)
+	rm -f migrate-curio
+	$(GOCC) build $(GOFLAGS) -o migrate-curio ./cmd/migrate-curio
+.PHONY: boostci
+
 react: validate-node-version
 	npm_config_legacy_peer_deps=yes npm ci --no-audit --prefix react
 	npm run --prefix react build
@@ -143,7 +148,7 @@ update-react: validate-node-version
 	npm run --prefix react build
 .PHONY: react
 
-build-go: boost boostd boostx boostd-data booster-http booster-bitswap devnet migrate-lid
+build-go: boost boostd boostx boostd-data booster-http booster-bitswap devnet migrate-lid migrate-curio
 .PHONY: build-go
 
 build: react build-go
@@ -162,6 +167,7 @@ install-boost:
 	install -C ./booster-http /usr/local/bin/booster-http
 	install -C ./booster-bitswap /usr/local/bin/booster-bitswap
 	install -C ./migrate-lid /usr/local/bin/migrate-lid
+	install -C ./migrate-curio /usr/local/bin/migrate-curio
 
 install-devnet:
 	install -C ./devnet /usr/local/bin/devnet

api/api.go

@@ -26,6 +26,7 @@ type Boost interface {
 	Net
 
 	// MethodGroup: Boost
+	BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error)                                                                               //perm:admin
 	BoostIndexerAnnounceAllDeals(ctx context.Context) error                                                                                     //perm:admin
 	BoostIndexerListMultihashes(ctx context.Context, contextID []byte) ([]multihash.Multihash, error)                                           //perm:admin
 	BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error)                                                                            //perm:admin

cmd/boostd/index.go

@@ -22,6 +22,7 @@ var indexProvCmd = &cli.Command{
 		indexProvAnnounceLatestHttp,
 		indexProvAnnounceDealRemovalAd,
 		indexProvAnnounceDeal,
+		indexProvRemoveAllCmd,
 	},
 }
 
@@ -303,3 +304,29 @@ var indexProvAnnounceDeal = &cli.Command{
 		return nil
 	},
 }
+
+var indexProvRemoveAllCmd = &cli.Command{
+	Name:  "remove-all",
+	Usage: "Announce all removal ad for all contextIDs",
+	Flags: []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+
+		// get boost api
+		napi, closer, err := bcli.GetBoostAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		// announce markets and boost deals
+		cids, err := napi.BoostIndexerRemoveAll(ctx)
+		if err != nil {
+			return err
+		}
+		for _, c := range cids {
+			fmt.Println("Published the removal ad with CID", c.String())
+		}
+		return nil
+	},
+}

cmd/migrate-curio/leveldb.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+)
+
+var cleanupLevelDBCmd = &cli.Command{
+	Name:        "leveldb",
+	Description: "Removes the indexes and other metadata leveldb based LID store",
+	Usage:       "migrate-curio cleanup leveldb",
+	Before:      before,
+	Action: func(cctx *cli.Context) error {
+		fmt.Println("Please remove the directory called 'LID' in the boost repo path to remove leveldb based LID")
+		fmt.Println("This directory can also be present outside of Boost repo if 'boostd-data' was running with a custom repo path")
+		return nil
+	},
+}

cmd/migrate-curio/main.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"os"
+
+	"github.com/filecoin-project/boost/build"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/urfave/cli/v2"
+)
+
+var log = logging.Logger("migrate-curio")
+
+const (
+	FlagBoostRepo = "boost-repo"
+)
+
+var FlagRepo = &cli.StringFlag{
+	Name:    FlagBoostRepo,
+	EnvVars: []string{"BOOST_PATH"},
+	Usage:   "boost repo path",
+	Value:   "~/.boost",
+}
+
+var IsVeryVerbose bool
+
+var FlagVeryVerbose = &cli.BoolFlag{
+	Name:        "vv",
+	Usage:       "enables very verbose mode, useful for debugging the CLI",
+	Destination: &IsVeryVerbose,
+}
+
+func main() {
+	app := &cli.App{
+		Name:                 "migrate-curio",
+		Usage:                "Migrate boost to Curio",
+		EnableBashCompletion: true,
+		Version:              build.UserVersion(),
+		Flags: []cli.Flag{
+			FlagRepo,
+			FlagVeryVerbose,
+			&cli.StringFlag{
+				Name:    "db-host",
+				EnvVars: []string{"CURIO_DB_HOST", "CURIO_HARMONYDB_HOSTS"},
+				Usage:   "Command separated list of hostnames for yugabyte cluster",
+				Value:   "127.0.0.1",
+			},
+			&cli.StringFlag{
+				Name:    "db-name",
+				EnvVars: []string{"CURIO_DB_NAME", "CURIO_HARMONYDB_NAME"},
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-user",
+				EnvVars: []string{"CURIO_DB_USER", "CURIO_HARMONYDB_USERNAME"},
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-password",
+				EnvVars: []string{"CURIO_DB_PASSWORD", "CURIO_HARMONYDB_PASSWORD"},
+				Value:   "yugabyte",
+			},
+			&cli.StringFlag{
+				Name:    "db-port",
+				EnvVars: []string{"CURIO_DB_PORT", "CURIO_HARMONYDB_PORT"},
+				Value:   "5433",
+			},
+		},
+		Commands: []*cli.Command{
+			migrateCmd,
+			cleanupLIDCmd,
+		},
+	}
+	app.Setup()
+
+	if err := app.Run(os.Args); err != nil {
+		os.Stderr.WriteString("Error: " + err.Error() + "\n")
+	}
+}
+
+func before(cctx *cli.Context) error {
+	_ = logging.SetLogLevel("migrate-curio", "INFO")
+
+	if IsVeryVerbose {
+		_ = logging.SetLogLevel("migrate-curio", "DEBUG")
+	}
+
+	return nil
+}