Skip to content

[content-service] Make the name of the usage report bucket configurable #11689

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 2, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions components/content-service-api/go/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,13 @@ type PProf struct {
Addr string `json:"address"`
}

// UsageReportConfig configures the upload of workspace instance usage reports
type UsageReportConfig struct {
BucketName string `json:"bucketName"`
}

type ServiceConfig struct {
Service baseserver.ServerConfiguration `json:"service"`
Storage StorageConfig `json:"storage"`
Service baseserver.ServerConfiguration `json:"service"`
Storage StorageConfig `json:"storage"`
UsageReports UsageReportConfig `json:"usageReport"`
}
2 changes: 1 addition & 1 deletion components/content-service/cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ var runCmd = &cobra.Command{
}
api.RegisterIDEPluginServiceServer(srv.GRPC(), idePluginService)

usageReportService, err := service.NewUsageReportService(cfg.Storage)
usageReportService, err := service.NewUsageReportService(cfg.Storage, cfg.UsageReports.BucketName)
if err != nil {
log.WithError(err).Fatalf("Cannot create usage report service")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,25 +18,22 @@ import (
"github.com/gitpod-io/gitpod/content-service/pkg/storage"
)

const (
usageReportBucketName = "usage-reports"
)

// UsageReportService implements UsageReportServiceServer
type UsageReportService struct {
cfg config.StorageConfig
s storage.PresignedAccess
cfg config.StorageConfig
s storage.PresignedAccess
bucketName string

api.UnimplementedUsageReportServiceServer
}

// NewUsageReportService create a new usagereport service
func NewUsageReportService(cfg config.StorageConfig) (res *UsageReportService, err error) {
func NewUsageReportService(cfg config.StorageConfig, bucketName string) (res *UsageReportService, err error) {
s, err := storage.NewPresignedAccess(&cfg)
if err != nil {
return nil, err
}
return &UsageReportService{cfg: cfg, s: s}, nil
return &UsageReportService{cfg: cfg, s: s, bucketName: bucketName}, nil
}

// UploadURL provides a URL to which clients can upload the content via HTTP PUT.
Expand All @@ -45,17 +42,17 @@ func (us *UsageReportService) UploadURL(ctx context.Context, req *api.UsageRepor
span.SetTag("name", req.Name)
defer tracing.FinishSpan(span, &err)

err = us.s.EnsureExists(ctx, usageReportBucketName)
err = us.s.EnsureExists(ctx, us.bucketName)
if err != nil {
return nil, status.Error(codes.NotFound, err.Error())
}

info, err := us.s.SignUpload(ctx, usageReportBucketName, req.Name, &storage.SignedURLOptions{
info, err := us.s.SignUpload(ctx, us.bucketName, req.Name, &storage.SignedURLOptions{
ContentType: "*/*",
})
if err != nil {
log.WithField("name", req.Name).
WithField("bucket", usageReportBucketName).
WithField("bucket", us.bucketName).
WithError(err).
Error("Error getting UsageReport SignUpload URL")
return nil, status.Error(codes.Unknown, err.Error())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,20 @@ import (
// TestUploadURL tests that usageReportService.UploadURL interacts with PresignedAccess
// correctly to produce an upload URL for the correct bucket and filename.
func TestUploadURL(t *testing.T) {
const (
fileName = "some-report-filename"
bucketName = "gitpod-usage-reports"
)

ctrl := gomock.NewController(t)
s := storagemock.NewMockPresignedAccess(ctrl)
const fileName = "some-report-filename"

s.EXPECT().EnsureExists(gomock.Any(), usageReportBucketName).
s.EXPECT().EnsureExists(gomock.Any(), bucketName).
Return(nil)
s.EXPECT().SignUpload(gomock.Any(), usageReportBucketName, fileName, gomock.Any()).
s.EXPECT().SignUpload(gomock.Any(), bucketName, fileName, gomock.Any()).
Return(&storage.UploadInfo{URL: "http://example.com/some-path"}, nil)

svc := &UsageReportService{cfg: config.StorageConfig{}, s: s}
svc := &UsageReportService{cfg: config.StorageConfig{}, s: s, bucketName: bucketName}
resp, err := svc.UploadURL(context.Background(), &api.UsageReportUploadURLRequest{Name: fileName})

require.NoError(t, err)
Expand Down
12 changes: 12 additions & 0 deletions install/installer/pkg/components/content-service/configmap.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,30 @@ import (

"github.com/gitpod-io/gitpod/content-service/api/config"
"github.com/gitpod-io/gitpod/installer/pkg/common"
"github.com/gitpod-io/gitpod/installer/pkg/config/v1/experimental"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
)

func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
usageReportBucketName := "gitpod-usage-reports"
_ = ctx.WithExperimental(func(cfg *experimental.Config) error {
if cfg.Workspace != nil && cfg.Workspace.ContentService.UsageReportBucketName != "" {
usageReportBucketName = cfg.Workspace.ContentService.UsageReportBucketName
}
return nil
})

cscfg := config.ServiceConfig{
Service: baseserver.ServerConfiguration{
Address: fmt.Sprintf(":%d", RPCPort),
},
Storage: common.StorageConfig(ctx),
UsageReports: config.UsageReportConfig{
BucketName: usageReportBucketName,
},
}

fc, err := common.ToJSONString(cscfg)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
// Licensed under the MIT License. See License-MIT.txt in the project root for license information.

package content_service

import (
"encoding/json"
"testing"

csconfig "github.com/gitpod-io/gitpod/content-service/api/config"
"github.com/gitpod-io/gitpod/installer/pkg/common"
"github.com/gitpod-io/gitpod/installer/pkg/config/v1"
"github.com/gitpod-io/gitpod/installer/pkg/config/v1/experimental"
"github.com/gitpod-io/gitpod/installer/pkg/config/versions"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
"k8s.io/utils/pointer"
)

func TestConfigMap_CanConfigureUsageReportBucketName(t *testing.T) {
ctx := newRenderContext(t)
objs, err := configmap(ctx)
require.NoError(t, err)
require.Len(t, objs, 1, "must only render one configmap")

expectedUsageReportConfig := csconfig.UsageReportConfig{BucketName: "some-bucket-name"}
expectedJSON, err := common.ToJSONString(expectedUsageReportConfig)
require.NoError(t, err)

cm, ok := objs[0].(*corev1.ConfigMap)
require.True(t, ok)

var fullConfig csconfig.ServiceConfig
err = json.Unmarshal([]byte(cm.Data["config.json"]), &fullConfig)
require.NoError(t, err)

actualUsageReportConfig := fullConfig.UsageReports
actualJSON, err := common.ToJSONString(actualUsageReportConfig)
require.NoError(t, err)

require.JSONEq(t, string(expectedJSON), string(actualJSON))
}

func newRenderContext(t *testing.T) *common.RenderContext {
t.Helper()

ctx, err := common.NewRenderContext(config.Config{
Domain: "test.domain.everything.awesome.is",
ObjectStorage: config.ObjectStorage{InCluster: pointer.Bool(true)},
Experimental: &experimental.Config{
Workspace: &experimental.WorkspaceConfig{
ContentService: struct {
UsageReportBucketName string "json:\"usageReportBucketName\""
}{UsageReportBucketName: "some-bucket-name"},
},
},
}, versions.Manifest{}, "test-namespace")

require.NoError(t, err)

return ctx
}
4 changes: 4 additions & 0 deletions install/installer/pkg/config/v1/experimental/experimental.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,10 @@ type WorkspaceConfig struct {
GitpodInstallationWorkspaceHostSuffix string `json:"gitpodInstallationWorkspaceHostSuffix"`
GitpodInstallationWorkspaceHostSuffixRegex string `json:"gitpodInstallationWorkspaceHostSuffixRegex"`
} `json:"wsProxy"`

ContentService struct {
UsageReportBucketName string `json:"usageReportBucketName"`
} `json:"contentService"`
}

type PersistentVolumeClaim struct {
Expand Down