feat: add blob storage service (#1672)

Co-authored-by: colin <102356659+colinlyguo@users.noreply.github.com>
Co-authored-by: yiweichi <yiweichi@users.noreply.github.com>

Author: yiweichi

.github/workflows/docker.yml

@@ -99,6 +99,51 @@ jobs:
             ${{ env.ECR_REGISTRY }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }}
             ${{ env.ECR_REGISTRY }}/${{ env.REPOSITORY }}:latest
 
+  blob_uploader:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+      - name: check repo and create it if not exist
+        env:
+          REPOSITORY: blob-uploader
+        run: |
+          aws --region ${{ env.AWS_REGION }} ecr describe-repositories --repository-names ${{ env.REPOSITORY }} && : || aws --region ${{ env.AWS_REGION }} ecr create-repository --repository-name ${{ env.REPOSITORY }}
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          REPOSITORY: blob-uploader
+          IMAGE_TAG: ${{ github.ref_name }}
+        with:
+          context: .
+          file: ./build/dockerfiles/blob_uploader.Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            scrolltech/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }}
+            scrolltech/${{ env.REPOSITORY }}:latest
+            ${{ env.ECR_REGISTRY }}/${{ env.REPOSITORY }}:${{ env.IMAGE_TAG }}
+            ${{ env.ECR_REGISTRY }}/${{ env.REPOSITORY }}:latest
+
   rollup-db-cli:
     runs-on: ubuntu-latest
     steps:

build/dockerfiles/blob_uploader.Dockerfile

@@ -0,0 +1,30 @@
+# Download Go dependencies
+FROM scrolltech/go-rust-builder:go-1.22-rust-nightly-2023-12-03 as base
+
+WORKDIR /src
+COPY go.work* ./
+COPY ./rollup/go.* ./rollup/
+COPY ./common/go.* ./common/
+COPY ./coordinator/go.* ./coordinator/
+COPY ./database/go.* ./database/
+COPY ./tests/integration-test/go.* ./tests/integration-test/
+COPY ./bridge-history-api/go.* ./bridge-history-api/
+RUN go mod download -x
+
+# Build blob_uploader
+FROM base as builder
+
+RUN --mount=target=. \
+    --mount=type=cache,target=/root/.cache/go-build \
+    cd /src/rollup/cmd/blob_uploader/ && CGO_LDFLAGS="-ldl" go build -v -p 4 -o /bin/blob_uploader
+
+# Pull blob_uploader into a second stage deploy ubuntu container
+FROM ubuntu:20.04
+
+RUN apt update && apt install vim netcat-openbsd net-tools curl ca-certificates -y
+
+ENV CGO_LDFLAGS="-ldl"
+
+COPY --from=builder /bin/blob_uploader /bin/
+WORKDIR /app
+ENTRYPOINT ["blob_uploader"]

build/dockerfiles/blob_uploader.Dockerfile.dockerignore

@@ -0,0 +1,5 @@
+assets/
+docs/
+l2geth/
+rpc-gateway/
+*target/*

common/testdata/blobdata.json

@@ -326,3 +326,53 @@ func (s TxStatus) String() string {
 		return fmt.Sprintf("Unknown TxStatus (%d)", int32(s))
 	}
 }
+
+// BlobUploadStatus represents the status of a blob upload
+type BlobUploadStatus int
+
+const (
+	// BlobUploadStatusUndefined indicates an undefined status
+	BlobUploadStatusUndefined BlobUploadStatus = iota
+	// BlobUploadStatusPending indicates a pending upload status
+	BlobUploadStatusPending
+	// BlobUploadStatusUploaded indicates a successful upload status
+	BlobUploadStatusUploaded
+	// BlobUploadStatusFailed indicates a failed upload status
+	BlobUploadStatusFailed
+)
+
+func (s BlobUploadStatus) String() string {
+	switch s {
+	case BlobUploadStatusPending:
+		return "BlobUploadStatusPending"
+	case BlobUploadStatusUploaded:
+		return "BlobUploadStatusUploaded"
+	case BlobUploadStatusFailed:
+		return "BlobUploadStatusFailed"
+	default:
+		return fmt.Sprintf("Unknown BlobUploadStatus (%d)", int32(s))
+	}
+}
+
+// BlobStoragePlatform represents the platform a blob upload to
+type BlobStoragePlatform int
+
+const (
+	// BlobStoragePlatformUndefined indicates an undefined platform
+	BlobStoragePlatformUndefined BlobStoragePlatform = iota
+	// BlobStoragePlatformS3 represents AWS S3
+	BlobStoragePlatformS3
+	// BlobStoragePlatformArweave represents storage blockchain Arweave
+	BlobStoragePlatformArweave
+)
+
+func (s BlobStoragePlatform) String() string {
+	switch s {
+	case BlobStoragePlatformS3:
+		return "BlobStoragePlatformS3"
+	case BlobStoragePlatformArweave:
+		return "BlobStoragePlatformArweave"
+	default:
+		return fmt.Sprintf("Unknown BlobStoragePlatform (%d)", int32(s))
+	}
+}

common/types/db.go

@@ -0,0 +1,23 @@
+package utils
+
+import (
+	"crypto/sha256"
+	"fmt"
+
+	"github.com/scroll-tech/go-ethereum/crypto/kzg4844"
+)
+
+// CalculateVersionedBlobHash calculate the kzg4844 versioned blob hash from a blob
+func CalculateVersionedBlobHash(blob kzg4844.Blob) ([32]byte, error) {
+	// calculate kzg4844 commitment from blob
+	commit, err := kzg4844.BlobToCommitment(&blob)
+	if err != nil {
+		return [32]byte{}, fmt.Errorf("failed to get blob commitment, err: %w", err)
+	}
+
+	// calculate kzg4844 versioned blob hash from blob commitment
+	hasher := sha256.New()
+	vh := kzg4844.CalcBlobHashV1(hasher, &commit)
+
+	return vh, nil
+}

common/utils/blob.go

@@ -0,0 +1,51 @@
+package utils
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"os"
+	"testing"
+
+	"github.com/scroll-tech/go-ethereum/crypto/kzg4844"
+)
+
+type BlobData struct {
+	VersionedBlobHash string `json:"versionedBlobHash"`
+	BlobData          string `json:"blobData"`
+}
+
+// TestCalculateVersionedBlobHash tests the CalculateVersionedBlobHash function
+func TestCalculateVersionedBlobHash(t *testing.T) {
+	// Read the test data
+	data, err := os.ReadFile("../testdata/blobdata.json")
+	if err != nil {
+		t.Fatalf("Failed to read blobdata.json: %v", err)
+	}
+
+	var blobData BlobData
+	if err := json.Unmarshal(data, &blobData); err != nil {
+		t.Fatalf("Failed to parse blobdata.json: %v", err)
+	}
+
+	blobBytes, err := hex.DecodeString(blobData.BlobData)
+	if err != nil {
+		t.Fatalf("Failed to decode blob data: %v", err)
+	}
+
+	// Convert []byte to kzg4844.Blob
+	var blob kzg4844.Blob
+	copy(blob[:], blobBytes)
+
+	// Calculate the hash
+	calculatedHashBytes, err := CalculateVersionedBlobHash(blob)
+	if err != nil {
+		t.Fatalf("Failed to calculate versioned blob hash: %v", err)
+	}
+
+	calculatedHash := hex.EncodeToString(calculatedHashBytes[:])
+
+	if calculatedHash != blobData.VersionedBlobHash {
+		t.Fatalf("Hash mismatch: got %s, want %s", calculatedHash, blobData.VersionedBlobHash)
+	}
+
+}

common/utils/blob_test.go

@@ -5,7 +5,7 @@ import (
 	"runtime/debug"
 )
 
-var tag = "v4.5.21"
+var tag = "v4.5.22"
 
 var commit = func() string {
 	if info, ok := debug.ReadBuildInfo(); ok {

common/version/version.go

@@ -2,6 +2,7 @@ package types
 
 import (
 	"fmt"
+
 	"scroll-tech/common/types/message"
 )
 

coordinator/internal/types/prover.go

@@ -59,20 +59,20 @@ func testResetDB(t *testing.T) {
 	cur, err := Current(pgDB)
 	assert.NoError(t, err)
 	// total number of tables.
-	assert.Equal(t, int64(26), cur)
+	assert.Equal(t, int64(27), cur)
 }
 
 func testMigrate(t *testing.T) {
 	assert.NoError(t, Migrate(pgDB))
 	cur, err := Current(pgDB)
 	assert.NoError(t, err)
-	assert.Equal(t, int64(26), cur)
+	assert.Equal(t, int64(27), cur)
 }
 
 func testRollback(t *testing.T) {
 	version, err := Current(pgDB)
 	assert.NoError(t, err)
-	assert.Equal(t, int64(26), version)
+	assert.Equal(t, int64(27), version)
 
 	assert.NoError(t, Rollback(pgDB, nil))