Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

able to pass in path in bucket and trim hash #1

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ build:
.PHONY: build

build-docker:
docker-compose build
docker compose build
.PHONY: build-docker

clean:
Expand All @@ -21,8 +21,8 @@ test:
.PHONY: test

integration:
docker-compose down
docker-compose up -d minio create-buckets
docker compose down
docker compose up -d minio create-buckets
RUN_INTEGRATION_TESTS=true go test -v ./...
.PHONY: integration

Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ To run the project locally, you should first copy `.env.template` to `.env` and
to your beacon client and storage backend of choice. Then you can run the project with:

```sh
docker-compose up
docker compose up
```

You can see a full list of configuration options by running:
Expand Down
5 changes: 3 additions & 2 deletions archiver/flags/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package flags

import (
"fmt"
"strings"
"time"

common "github.com/base-org/blob-archiver/common/flags"
Expand Down Expand Up @@ -35,7 +36,7 @@ func (c ArchiverConfig) Check() error {
}

if c.OriginBlock == (geth.Hash{}) {
return fmt.Errorf("invalid origin block")
return fmt.Errorf("invalid origin block %s", c.OriginBlock)
}

if c.ListenAddr == "" {
Expand All @@ -53,7 +54,7 @@ func ReadConfig(cliCtx *cli.Context) ArchiverConfig {
BeaconConfig: common.NewBeaconConfig(cliCtx),
StorageConfig: common.NewStorageConfig(cliCtx),
PollInterval: pollInterval,
OriginBlock: geth.HexToHash(cliCtx.String(ArchiverOriginBlock.Name)),
OriginBlock: geth.HexToHash(strings.Trim(cliCtx.String(ArchiverOriginBlock.Name), "\"")),
ListenAddr: cliCtx.String(ArchiverListenAddrFlag.Name),
}
}
2 changes: 2 additions & 0 deletions common/flags/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ type S3Config struct {
Endpoint string
UseHttps bool
Bucket string
Path string

S3CredentialType S3CredentialType
AccessKey string
Expand Down Expand Up @@ -106,6 +107,7 @@ func readS3Config(ctx *cli.Context) S3Config {
SecretAccessKey: ctx.String(S3SecretAccessKeyFlagName),
UseHttps: ctx.Bool(S3EndpointHttpsFlagName),
Bucket: ctx.String(S3BucketFlagName),
Path: ctx.String(S3PathFlagName),
S3CredentialType: toS3CredentialType(ctx.String(S3CredentialTypeFlagName)),
Compress: ctx.Bool(S3CompressFlagName),
}
Expand Down
8 changes: 8 additions & 0 deletions common/flags/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ const (
S3AccessKeyFlagName = "s3-access-key"
S3SecretAccessKeyFlagName = "s3-secret-access-key"
S3BucketFlagName = "s3-bucket"
S3PathFlagName = "s3-path"
FileStorageDirectoryFlagName = "file-directory"
)

Expand Down Expand Up @@ -77,6 +78,13 @@ func CLIFlags(envPrefix string) []cli.Flag {
Hidden: true,
EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_BUCKET"),
},
&cli.StringFlag{
Name: S3PathFlagName,
Usage: "The path to append to file",
Hidden: true,
EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_PATH"),
Value: "",
},
// File Data Store Flags
&cli.StringFlag{
Name: FileStorageDirectoryFlagName,
Expand Down
9 changes: 6 additions & 3 deletions common/storage/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"context"
"encoding/json"
"io"
"path"

"github.com/base-org/blob-archiver/common/flags"
"github.com/ethereum/go-ethereum/common"
Expand All @@ -17,6 +18,7 @@ import (
type S3Storage struct {
s3 *minio.Client
bucket string
path string
log log.Logger
compress bool
}
Expand All @@ -41,13 +43,14 @@ func NewS3Storage(cfg flags.S3Config, l log.Logger) (*S3Storage, error) {
return &S3Storage{
s3: client,
bucket: cfg.Bucket,
path: cfg.Path,
log: l,
compress: cfg.Compress,
}, nil
}

func (s *S3Storage) Exists(ctx context.Context, hash common.Hash) (bool, error) {
_, err := s.s3.StatObject(ctx, s.bucket, hash.String(), minio.StatObjectOptions{})
_, err := s.s3.StatObject(ctx, s.bucket, path.Join(s.path, hash.String()), minio.StatObjectOptions{})
if err != nil {
errResponse := minio.ToErrorResponse(err)
if errResponse.Code == "NoSuchKey" {
Expand All @@ -61,7 +64,7 @@ func (s *S3Storage) Exists(ctx context.Context, hash common.Hash) (bool, error)
}

func (s *S3Storage) Read(ctx context.Context, hash common.Hash) (BlobData, error) {
res, err := s.s3.GetObject(ctx, s.bucket, hash.String(), minio.GetObjectOptions{})
res, err := s.s3.GetObject(ctx, s.bucket, path.Join(s.path, hash.String()), minio.GetObjectOptions{})
if err != nil {
s.log.Info("unexpected error fetching blob", "hash", hash.String(), "err", err)
return BlobData{}, ErrStorage
Expand Down Expand Up @@ -122,7 +125,7 @@ func (s *S3Storage) Write(ctx context.Context, data BlobData) error {

reader := bytes.NewReader(b)

_, err = s.s3.PutObject(ctx, s.bucket, data.Header.BeaconBlockHash.String(), reader, int64(len(b)), options)
_, err = s.s3.PutObject(ctx, s.bucket, path.Join(s.path, data.Header.BeaconBlockHash.String()), reader, int64(len(b)), options)

if err != nil {
s.log.Warn("error writing blob", "err", err)
Expand Down
4 changes: 2 additions & 2 deletions common/storage/s3_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ import (

// Prior to running these tests, a local Minio server must be running.
// You can accomplish this with:
// docker-compose down # shut down any running services
// docker-compose up minio create-buckets # start the minio service
// docker compose down # shut down any running services
// docker compose up minio create-buckets # start the minio service
func setupS3(t *testing.T) *S3Storage {
if os.Getenv("RUN_INTEGRATION_TESTS") == "" {
t.Skip("skipping integration tests: set RUN_INTEGRATION_TESTS environment variable")
Expand Down