Skip to content

Commit

Permalink
Merge pull request #59 from UNIwise/feature/MP2381/MP-2385-S3-storage
Browse files Browse the repository at this point in the history
Feature/mp2381/mp 2385 s3 storage
  • Loading branch information
IslamMonjid authored Aug 19, 2024
2 parents 18e5556 + 615ae27 commit 795205f
Show file tree
Hide file tree
Showing 11 changed files with 186 additions and 3 deletions.
3 changes: 3 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,6 @@ SERVER_PORT=9000

LOG_LEVEL: debug
LOG_FORMAT: text

AWS_REGION=eu-west-1
AWS_BUCKET_NAME=my-bucket
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ The server can be configured with a yaml configuration file specified with the `
| prometheus.path | expose prometheus metrics under path | string | `/metrics` |
| prometheus.port | port to expose the prometheus metrics under | int | `9090` |
| api.token | secret token to authenticating against poeditor | string |
| aws.region | AWS Region | string | `eu-west-1` |
| aws.bucket.name | AWS bucket name | string |

# API specification

Expand Down
22 changes: 20 additions & 2 deletions cmd/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
Expand All @@ -32,6 +32,7 @@ import (
"github.com/uniwise/parrot/internal/metrics"
"github.com/uniwise/parrot/internal/project"
"github.com/uniwise/parrot/internal/rest"
"github.com/uniwise/parrot/internal/storage"
"github.com/uniwise/parrot/pkg/poedit"
)

Expand Down Expand Up @@ -61,6 +62,9 @@ const (
confPrometheusPort = "prometheus.port"

confAPIToken = "api.token"

confAWSRegion = "aws.region"
confAWSBucket = "aws.bucket.name"
)

// serveCmd represents the serve command
Expand All @@ -80,7 +84,20 @@ by caching exports from poeditor`,

cli := poedit.NewClient(viper.GetString(confAPIToken), http.DefaultClient)

svc := project.NewService(cli, cacheInstance, viper.GetDuration(confCacheRenewalThreshold), logrus.NewEntry(logger))
s3Config := &storage.S3StorageConfig{
Region: viper.GetString(confAWSRegion),
Bucket: viper.GetString(confAWSBucket),
}

s3Client, err := storage.NewS3Client(context.Background(), *s3Config)

if err != nil {
logger.Fatal(err)
}

storageService := storage.NewService(context.Background(), s3Client)

svc := project.NewService(cli, storageService, cacheInstance, viper.GetDuration(confCacheRenewalThreshold), logrus.NewEntry(logger))

server, err := rest.NewServer(logrus.NewEntry(logger), svc, viper.GetBool(confPrometheusEnabled))
if err != nil {
Expand Down Expand Up @@ -138,6 +155,7 @@ func init() {
viper.SetDefault(confPrometheusEnabled, true)
viper.SetDefault(confPrometheusPort, 9090)
viper.SetDefault(confPrometheusPath, "/metrics")
viper.SetDefault(confAWSRegion, "eu-west-1")

rootCmd.AddCommand(serveCmd)
}
Expand Down
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ go 1.16

require (
github.com/AppsFlyer/go-sundheit v0.5.0
github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect
github.com/aws/aws-sdk-go-v2/config v1.27.27 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 // indirect
github.com/go-playground/universal-translator v0.17.0 // indirect
github.com/go-playground/validator v9.31.0+incompatible
github.com/go-redis/cache/v8 v8.4.4
Expand Down
39 changes: 39 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,42 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb
github.com/armon/go-metrics v0.4.0/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=
github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM=
github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90=
github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg=
github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI=
github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 h1:Z5r7SycxmSllHYmaAZPpmN8GviDrSGhMS6bldqtXZPw=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15/go.mod h1:CetW7bDE00QoGEmPUoZuRog07SGVAUVW6LFpNP0YfIg=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 h1:YPYe6ZmvUfDDDELqEKtAd6bo8zxhkm+XEFEzQisqUIE=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17/go.mod h1:oBtcnYua/CgzCWYN7NZ5j7PotFDaFSUjCYVTtfyn7vw=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 h1:246A4lSTXWJw/rmlQI+TT2OcqeDMKBdyjEQrafMaQdA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15/go.mod h1:haVfg3761/WF7YPuJOER2MP0k4UAXyHaLclKXB6usDg=
github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 h1:hT8ZAZRIfqBqHbzKTII+CIiY8G2oC9OpLedkZ51DWl8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3/go.mod h1:Lcxzg5rojyVPU/0eFwLtcyTaek/6Mtic5B1gJo7e/zE=
github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM=
github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw=
github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE=
github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ=
github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
Expand Down Expand Up @@ -422,6 +458,9 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/johngb/langreg v0.0.0-20150123211413-5c6abc6d19d2 h1:R0Yc1jK2pjDwZeIXmcbELtKLedE+PjuI0S5cguGxTxw=
github.com/johngb/langreg v0.0.0-20150123211413-5c6abc6d19d2/go.mod h1:m/usUv5KgruWsRUejHsR568dyOh5pJ1wVoKZKMuEPhI=
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
Expand Down
5 changes: 4 additions & 1 deletion internal/project/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/uniwise/parrot/internal/cache"
"github.com/uniwise/parrot/internal/storage"
"github.com/uniwise/parrot/pkg/poedit"
"golang.org/x/sync/semaphore"
)
Expand All @@ -34,15 +35,17 @@ type ServiceImpl struct {
Cache cache.Cache
RenewalThreshold time.Duration
PreFetchSemaphore *semaphore.Weighted
storage storage.Storage
}

func NewService(cli poedit.Client, cache cache.Cache, renewalThreshold time.Duration, entry *logrus.Entry) *ServiceImpl {
func NewService(cli poedit.Client, storage storage.Storage, cache cache.Cache, renewalThreshold time.Duration, entry *logrus.Entry) *ServiceImpl {
return &ServiceImpl{
Logger: entry,
Client: cli,
Cache: cache,
RenewalThreshold: renewalThreshold,
PreFetchSemaphore: semaphore.NewWeighted(1),
storage: storage,
}
}

Expand Down
6 changes: 6 additions & 0 deletions internal/storage/repository.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
//go:generate mockgen --source=repository.go -destination=repository_mock.go -package=storage
package storage

type Storage interface {
// TODO: Add methods to implement
}
32 changes: 32 additions & 0 deletions internal/storage/repository_mock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

35 changes: 35 additions & 0 deletions internal/storage/repository_s3.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package storage

import (
"context"

"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/pkg/errors"
)

type S3StorageConfig struct {
Region string `mapstructure:"region" validate:"required"`
Bucket string `mapstructure:"bucket" validate:"required"`
}

type S3ClientImpl struct {
config S3StorageConfig
client *s3.Client
}

func NewS3Client(ctx context.Context, conf S3StorageConfig) (*S3ClientImpl, error) {
cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(conf.Region))
if err != nil {
return nil, errors.Wrap(err, "failed to load configuration")
}

client := s3.NewFromConfig(cfg)

return &S3ClientImpl{
config: conf,
client: client,
}, nil
}

//TODO: Add methods to implement
18 changes: 18 additions & 0 deletions internal/storage/repository_s3_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package storage

import (
"context"
"testing"

"github.com/stretchr/testify/assert"
)

func TestNewS3Client(t *testing.T) {
t.Parallel()

client, err := NewS3Client(context.Background(), S3StorageConfig{})

assert.NoError(t, err)
assert.NotNil(t, client.client)
assert.NotNil(t, client.config)
}
23 changes: 23 additions & 0 deletions internal/storage/service.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package storage

import (
"context"

"github.com/aws/aws-sdk-go-v2/service/s3"
)

type ServiceImpl struct {
storage Storage
}

type Service interface {
ListObjectsV2(ctx context.Context) (*s3.ListObjectsV2Output, error)
}

func NewService(ctx context.Context, storage Storage) *ServiceImpl {
return &ServiceImpl{
storage: storage,
}
}

// TODO: Add methods to implement

0 comments on commit 795205f

Please sign in to comment.