Skip to content

Commit

Permalink
re-create bundle if verification failed
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexgao001
alexgao001 committed Apr 10, 2024
1 parent 17f59d4 commit 178dadf
Show file tree
Hide file tree
Showing 11 changed files with 188 additions and 63 deletions.
10 changes: 1 addition & 9 deletions cmd/blob-syncer/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package main

import (
"flag"
"fmt"
"github.com/bnb-chain/blob-syncer/config"
syncerdb "github.com/bnb-chain/blob-syncer/db"
"github.com/bnb-chain/blob-syncer/logging"
Expand All @@ -22,10 +21,6 @@ func initFlags() {
}
}

func printUsage() {
fmt.Print("usage: ./blob-syncer --config-path configFile\n")
}

func main() {
var (
cfg *config.SyncerConfig
Expand All @@ -36,10 +31,7 @@ func main() {
if configFilePath == "" {
configFilePath = os.Getenv(config.EnvVarConfigFilePath)

Check failure on line 32 in cmd/blob-syncer/main.go

View workflow job for this annotation

GitHub Actions / golangci-lint (1.20.x, ubuntu-latest) 

ineffectual assignment to configFilePath (ineffassign)
}
if configFilePath == "" {
printUsage()
return
}
configFilePath = "config/local/config-syncer.json"
cfg = config.ParseSyncerConfigFromFile(configFilePath)
if cfg == nil {
panic("failed to get configuration")
Expand Down
12 changes: 10 additions & 2 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,12 @@ type SyncerConfig struct {
}

func (s *SyncerConfig) Validate() {
if len(s.BucketName) == 0 {
panic("the Greenfield bucket name is not is not provided")
}
if s.StartSlot == 0 {
panic("the start slot to sync slot is not provided")
}
if len(s.BundleServiceEndpoints) == 0 {
panic("BundleService endpoints should not be empty")
}
Expand All @@ -37,7 +43,7 @@ func (s *SyncerConfig) Validate() {
panic("eth rpc address should not be empty")
}
if len(s.TempDir) == 0 {
panic("temp directory to hold files is missing")
panic("temp directory is not specified")
}
if len(s.PrivateKey) == 0 {
panic("private key is not provided")
Expand All @@ -63,6 +69,9 @@ type ServerConfig struct {
}

func (s *ServerConfig) Validate() {
if len(s.BucketName) == 0 {
panic("the Greenfield bucket name is not is not provided")
}
if len(s.BundleServiceEndpoints) == 0 {
panic("BundleService endpoints should not be empty")
}
Expand Down Expand Up @@ -152,7 +161,6 @@ func ParseServerConfigFromFile(filePath string) *ServerConfig {
if err != nil {
panic(err)
}

var config ServerConfig
if err = json.Unmarshal(bz, &config); err != nil {
panic(err)
Expand Down
4 changes: 2 additions & 2 deletions config/const.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@ const (
EnvVarDBUserName = "DB_USERNAME"
EnvVarDBUserPass = "DB_PASSWORD"
EnvVarPrivateKey = "PRIVATE_KEY"
)

const DefaultCreateBundleSlotInterval = 5
DefaultCreateBundleSlotInterval = 10
)
2 changes: 1 addition & 1 deletion db/bundle.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ const (

type Bundle struct {
Id int64
Name string `gorm:"NOT NULL;uniqueIndex:idx_bundle_name;size:96"`
Name string `gorm:"NOT NULL;uniqueIndex:idx_bundle_name;size:64"`
Status InnerBundleStatus `gorm:"NOT NULL"`
}

Expand Down
8 changes: 3 additions & 5 deletions db/dao.go
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package db

import (
"strings"

"gorm.io/gorm"
)

Expand Down Expand Up @@ -123,7 +121,7 @@ func (d *BlobSvcDB) GetLatestFinalizingBundle() (*Bundle, error) {
func (d *BlobSvcDB) CreateBundle(b *Bundle) error {
return d.db.Transaction(func(dbTx *gorm.DB) error {
err := dbTx.Create(b).Error
if err != nil && strings.Contains(err.Error(), " Duplicate entry") {
if err != nil && MysqlErrCode(err) != ErrDuplicateEntryCode {
return nil
}
return err
Expand All @@ -140,12 +138,12 @@ func (d *BlobSvcDB) UpdateBundleStatus(bundleName string, status InnerBundleStat
func (d *BlobSvcDB) SaveBlockAndBlob(block *Block, blobs []*Blob) error {
return d.db.Transaction(func(dbTx *gorm.DB) error {
err := dbTx.Save(block).Error
if err != nil {
if err != nil && MysqlErrCode(err) != ErrDuplicateEntryCode {
return err
}
if len(blobs) != 0 {
err = dbTx.Save(blobs).Error
if err != nil {
if err != nil && MysqlErrCode(err) != ErrDuplicateEntryCode {
return err
}
}
Expand Down
15 changes: 15 additions & 0 deletions db/type.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package db

import "github.com/go-sql-driver/mysql"

var (
ErrDuplicateEntryCode = 1062
)

func MysqlErrCode(err error) int {
mysqlErr, ok := err.(*mysql.MySQLError)
if !ok {
return 0
}
return int(mysqlErr.Number)
}
9 changes: 8 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ require (
github.com/go-openapi/strfmt v0.21.7
github.com/go-openapi/swag v0.22.4
github.com/go-openapi/validate v0.22.1
github.com/go-sql-driver/mysql v1.7.0
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d
github.com/jessevdk/go-flags v1.5.0
github.com/node-real/greenfield-bundle-service v0.0.1-beta
Expand All @@ -21,6 +22,7 @@ require (
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.15.0
golang.org/x/net v0.21.0
google.golang.org/api v0.107.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gorm.io/driver/mysql v1.5.1
gorm.io/driver/sqlite v1.5.3
Expand All @@ -29,6 +31,7 @@ require (
)

require (
cloud.google.com/go/compute/metadata v0.2.3 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
Expand All @@ -48,10 +51,12 @@ require (
github.com/go-openapi/analysis v0.21.4 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-sql-driver/mysql v1.7.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/uuid v1.5.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect
Expand Down Expand Up @@ -93,9 +98,11 @@ require (
github.com/tklauser/numcpus v0.7.0 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
go.mongodb.org/mongo-driver v1.11.3 // indirect
go.opencensus.io v0.24.0 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/oauth2 v0.12.0 // indirect
golang.org/x/sync v0.5.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
Expand Down
19 changes: 19 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,16 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb
cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY=
cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
cloud.google.com/go/compute v1.23.1 h1:V97tBoDaZHb6leicZ1G6DLK2BAaZLJ/7+9BB/En3hR0=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
Expand Down Expand Up @@ -190,6 +194,8 @@ github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
Expand Down Expand Up @@ -226,6 +232,7 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
Expand All @@ -249,8 +256,12 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.2.1 h1:RY7tHKZcRlk788d5WSo/e83gOyyy742E8GSs771ySpg=
github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ=
github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
Expand Down Expand Up @@ -413,6 +424,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
Expand Down Expand Up @@ -454,6 +466,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
Expand Down Expand Up @@ -532,6 +546,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
Expand All @@ -548,6 +563,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down Expand Up @@ -703,6 +720,8 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513
google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
google.golang.org/api v0.107.0 h1:I2SlFjD8ZWabaIFOfeEDg3pf0BHJDh6iYQ1ic3Yu/UU=
google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
Expand Down
42 changes: 23 additions & 19 deletions syncer/syncer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,13 +149,12 @@ func (s *BlobSyncer) process() error {
return err
}
}
err = s.writeBlobToFile(nextSlot, sideCars)
if err != nil {
if err = s.writeBlobToFile(nextSlot, bundleName, sideCars); err != nil {
return err
}

if nextSlot == s.bundleDetail.finalizeSlot {
err = s.finalizeBundle(bundleName)
err = s.finalizeCurBundle(bundleName)
if err != nil {
return err
}
Expand Down Expand Up @@ -213,9 +212,10 @@ func (s *BlobSyncer) calNextSlot() (uint64, error) {

// createLocalBundleDir creates an empty dir to hold blob files among a range of blocks, the blobs in this dir will be assembled into a bundle and uploaded to bundle service
func (s *BlobSyncer) createLocalBundleDir() error {
_, err := os.Stat(s.getBundleDir())
bundleName := s.bundleDetail.name
_, err := os.Stat(s.getBundleDir(bundleName))
if os.IsNotExist(err) {
err = os.MkdirAll(filepath.Dir(s.getBundleDir()), os.ModePerm)
err = os.MkdirAll(filepath.Dir(s.getBundleDir(bundleName)), os.ModePerm)
if err != nil {
return err
}
Expand All @@ -226,52 +226,56 @@ func (s *BlobSyncer) createLocalBundleDir() error {
Status: db.Finalizing,
})
}

func (s *BlobSyncer) finalizeBundle(bundleName string) error {
err := s.bundleClient.UploadAndFinalizeBundle(bundleName, s.getBucketName(), s.getBundleDir(), s.getBundleFilePath())
func (s *BlobSyncer) finalizeBundle(bundleName, bundleDir, bundleFilePath string) error {
err := s.bundleClient.UploadAndFinalizeBundle(bundleName, s.getBucketName(), bundleDir, bundleFilePath)
if err != nil {
if !strings.Contains(err.Error(), "Object exists") && !strings.Contains(err.Error(), "empty bundle") {
return err
}
}
err = os.RemoveAll(s.getBundleDir())
err = os.RemoveAll(bundleDir)
if err != nil {
return err
}
err = os.Remove(s.getBundleFilePath())
err = os.Remove(bundleFilePath)
if err != nil && !os.IsNotExist(err) {
return err
}
return s.blobDao.UpdateBundleStatus(bundleName, db.Finalized)
}

func (s *BlobSyncer) writeBlobToFile(slot uint64, blobs []*structs.Sidecar) error {
func (s *BlobSyncer) finalizeCurBundle(bundleName string) error {
return s.finalizeBundle(bundleName, s.getBundleDir(bundleName), s.getBundleFilePath(bundleName))
}

func (s *BlobSyncer) writeBlobToFile(slot uint64, bundleName string, blobs []*structs.Sidecar) error {
for i, b := range blobs {
blobName := types.GetBlobName(slot, i)
file, err := os.Create(s.getBlobPath(blobName))
file, err := os.Create(s.getBlobPath(bundleName, blobName))
if err != nil {
logging.Logger.Errorf("failed to create file, err=%s", err.Error())
return err
}
defer file.Close()
_, err = file.WriteString(b.Blob)
if err != nil {
return fmt.Errorf("failed to write string, err=%s", err.Error())
logging.Logger.Errorf("failed to write string, err=%s", err.Error())
return err
}
}
return nil
}

func (s *BlobSyncer) getBundleDir() string {
return fmt.Sprintf("%s/%s/", s.config.TempDir, s.bundleDetail.name)
func (s *BlobSyncer) getBundleDir(bundleName string) string {
return fmt.Sprintf("%s/%s/", s.config.TempDir, bundleName)
}

func (s *BlobSyncer) getBlobPath(blobName string) string {
return fmt.Sprintf("%s/%s/%s", s.config.TempDir, s.bundleDetail.name, blobName)
func (s *BlobSyncer) getBlobPath(bundleName, blobName string) string {
return fmt.Sprintf("%s/%s/%s", s.config.TempDir, bundleName, blobName)
}

func (s *BlobSyncer) getBundleFilePath() string {
return fmt.Sprintf("%s/%s.bundle", s.config.TempDir, s.bundleDetail.name)
func (s *BlobSyncer) getBundleFilePath(bundleName string) string {
return fmt.Sprintf("%s/%s.bundle", s.config.TempDir, bundleName)
}

func (s *BlobSyncer) LoadProgressAndResume(nextSlot uint64) error {
Expand Down
Loading

0 comments on commit 178dadf

Please sign in to comment.