export converter as a usable package

README.md

@@ -136,10 +136,11 @@ Acceleration Service Framework provides a built-in extensible method called driv
 type Driver interface {
 	// Convert converts the source image to target image, where
 	// content parameter provides necessary image utils, image
-	// content store and so on. If conversion successful, the
+	// content store and so on, where source parameter is the
+	// original image reference. If conversion successful, the
 	// converted image manifest will be returned, otherwise a
 	// non-nil error will be returned.
-	Convert(context.Context, content.Provider) (*ocispec.Descriptor, error)
+	Convert(ctx context.Context, content content.Provider, source string) (*ocispec.Descriptor, error)
 
 	// Name gets the driver type name, it is used to identify
 	// different accelerated image formats.

go.mod

@@ -29,6 +29,25 @@ require (
 	github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/astaxie/beego v1.12.1 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.17.2 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.18.4 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.4 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.20 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.43 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.27 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.20 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.29.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.11.26 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.17.6 // indirect
+	github.com/aws/smithy-go v1.13.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
@@ -54,6 +73,7 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/gomodule/redigo v2.0.0+incompatible // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/klauspost/compress v1.15.12 // indirect
 	github.com/lib/pq v1.10.0 // indirect
 	github.com/mattn/go-colorable v0.1.11 // indirect
@@ -101,4 +121,4 @@ require (
 )
 
 // It will be updated to official repo once nydus-snapshotter release.
-replace github.com/containerd/nydus-snapshotter => github.com/imeoer/nydus-snapshotter v0.3.20
+replace github.com/containerd/nydus-snapshotter => github.com/imeoer/nydus-snapshotter v0.3.22

go.sum

@@ -177,6 +177,44 @@ github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
+github.com/aws/aws-sdk-go-v2 v1.17.2 h1:r0yRZInwiPBNpQ4aDy/Ssh3ROWsGtKDwar2JS8Lm+N8=
+github.com/aws/aws-sdk-go-v2 v1.17.2/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno=
+github.com/aws/aws-sdk-go-v2/config v1.18.4 h1:VZKhr3uAADXHStS/Gf9xSYVmmaluTUfkc0dcbPiDsKE=
+github.com/aws/aws-sdk-go-v2/config v1.18.4/go.mod h1:EZxMPLSdGAZ3eAmkqXfYbRppZJTzFTkv8VyEzJhKko4=
+github.com/aws/aws-sdk-go-v2/credentials v1.13.4 h1:nEbHIyJy7mCvQ/kzGG7VWHSBpRB4H6sJy3bWierWUtg=
+github.com/aws/aws-sdk-go-v2/credentials v1.13.4/go.mod h1:/Cj5w9LRsNTLSwexsohwDME32OzJ6U81Zs33zr2ZWOM=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.20 h1:tpNOglTZ8kg9T38NpcGBxudqfUAwUzyUnLQ4XSd0CHE=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.20/go.mod h1:d9xFpWd3qYwdIXM0fvu7deD08vvdRXyc/ueV+0SqaWE=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.43 h1:+bkAMTd5OGyHu2nwNOangjEsP65fR0uhMbZJA52sZ64=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.43/go.mod h1:sS2tu0VEspKuY5eM1vQgy7P/hpZX8F62o6qsghZExWc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26 h1:5WU31cY7m0tG+AiaXuXGoMzo2GBQ1IixtWa8Yywsgco=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26/go.mod h1:2E0LdbJW6lbeU4uxjum99GZzI0ZjDpAb0CoSCM0oeEY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20 h1:WW0qSzDWoiWU2FS5DbKpxGilFVlCEJPwx4YtjdfI0Jw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20/go.mod h1:/+6lSiby8TBFpTVXZgKiN/rCfkYXEGvhlM4zCgPpt7w=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.3.27 h1:N2eKFw2S+JWRCtTt0IhIX7uoGGQciD4p6ba+SJv4WEU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.3.27/go.mod h1:RdwFVc7PBYWY33fa2+8T1mSqQ7ZEK4ILpM0wfioDC3w=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17 h1:5tXbMJ7Jq0iG65oiMg6tCLsHkSaO2xLXa2EmZ29vaTA=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17/go.mod h1:twV0fKMQuqLY4klyFH56aXNq3AFiA5LO0/frTczEOFE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21 h1:77b1GfaSuIok5yB/3HYbG+ypWvOJDQ2rVdq943D17R4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21/go.mod h1:sPOz31BVdqeeurKEuUpLNSve4tdCNPluE+070HNcEHI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.20 h1:jlgyHbkZQAgAc7VIxJDmtouH8eNjOk2REVAQfVhdaiQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.20/go.mod h1:Xs52xaLBqDEKRcAfX/hgjmD3YQ7c/W+BEyfamlO/W2E=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20 h1:4K6dbmR0mlp3o4Bo78PnpvzHtYAqEeVMguvEenpMGsI=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20/go.mod h1:1XpDcReIEOHsjwNToDKhIAO3qwLo1BnfbtSqWJa8j7g=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.29.5 h1:nRSEQj1JergKTVc8RGkhZvOEGgcvo4fWpDPwGDeg2ok=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.29.5/go.mod h1:wcaJTmjKFDW0s+Se55HBNIds6ghdAGoDDw+SGUdrfAk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.11.26 h1:ActQgdTNQej/RuUJjB9uxYVLDOvRGtUreXF8L3c8wyg=
+github.com/aws/aws-sdk-go-v2/service/sso v1.11.26/go.mod h1:uB9tV79ULEZUXc6Ob18A46KSQ0JDlrplPni9XW6Ot60=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.9 h1:wihKuqYUlA2T/Rx+yu2s6NDAns8B9DgnRooB1PVhY+Q=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.9/go.mod h1:2E/3D/mB8/r2J7nK42daoKP/ooCwbf0q1PznNc+DZTU=
+github.com/aws/aws-sdk-go-v2/service/sts v1.17.6 h1:VQFOLQVL3BrKM/NLO/7FiS4vcp5bqK0mGMyk09xLoAY=
+github.com/aws/aws-sdk-go-v2/service/sts v1.17.6/go.mod h1:Az3OXXYGyfNwQNsK/31L4R75qFYnO641RZGAoV3uH1c=
+github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
+github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
 github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd/go.mod h1:1b+Y/CofkYwXMUU0OhQqGvsY2Bvgr4j6jfT699wyZKQ=
@@ -851,8 +889,8 @@ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/imeoer/nydus-snapshotter v0.3.20 h1:kjkZpPg5hYgmrCVkCHgrS+QVVwFCv8qcumml53NnuJc=
-github.com/imeoer/nydus-snapshotter v0.3.20/go.mod h1:P2SwdbwhVg8A2RkG3EjpZ6STXYZcUa8nl411rkqHaLw=
+github.com/imeoer/nydus-snapshotter v0.3.22 h1:Vg+dKVTmmiNGWCJ/4KMCYHIgAx3demSPXvxnBLv2UJ0=
+github.com/imeoer/nydus-snapshotter v0.3.22/go.mod h1:xRVCi0yITol7k6yzjGY1tdlYZZVy5Qn+7YVLDjZ+0AE=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
 github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ=
@@ -888,7 +926,9 @@ github.com/jinzhu/now v1.0.0/go.mod h1:oHTiXerJ20+SfYcrdlBO7rzZRJWGwSTQ0iUY2jI6G
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=

pkg/adapter/adapter.go

@@ -0,0 +1,151 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package adapter
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/containerd/containerd"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+
+	"github.com/goharbor/acceleration-service/pkg/config"
+	"github.com/goharbor/acceleration-service/pkg/content"
+	"github.com/goharbor/acceleration-service/pkg/converter"
+	"github.com/goharbor/acceleration-service/pkg/errdefs"
+	"github.com/goharbor/acceleration-service/pkg/metrics"
+	"github.com/goharbor/acceleration-service/pkg/task"
+)
+
+type Adapter interface {
+	// Dispatch dispatches a conversion task to worker queue
+	// by specifying source image reference, the conversion is
+	// asynchronous, and if the sync option is specified,
+	// Dispatch will be blocked until the conversion is complete.
+	Dispatch(ctx context.Context, ref string, sync bool) error
+	// CheckHealth checks the containerd client can successfully
+	// connect to the containerd daemon and the healthcheck service
+	// returns the SERVING response.
+	CheckHealth(ctx context.Context) error
+}
+
+type LocalAdapter struct {
+	cfg    *config.Config
+	client *containerd.Client
+	rule   *Rule
+	worker *Worker
+	cvt    *converter.LocalConverter
+}
+
+func NewLocalAdapter(cfg *config.Config) (*LocalAdapter, error) {
+	client, err := containerd.New(
+		cfg.Provider.Containerd.Address,
+		containerd.WithDefaultNamespace("harbor-acceleration-service"),
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "create containerd client")
+	}
+
+	provider, err := content.NewLocalProvider(client, cfg.Host)
+	if err != nil {
+		return nil, errors.Wrap(err, "create content provider")
+	}
+
+	cvt, err := converter.NewLocalConverter(
+		converter.WithProvider(provider),
+		converter.WithDriver(cfg.Converter.Driver.Type, cfg.Converter.Driver.Config),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	worker, err := NewWorker(cfg.Converter.Worker)
+	if err != nil {
+		return nil, errors.Wrap(err, "create worker")
+	}
+
+	rule := &Rule{
+		items: cfg.Converter.Rules,
+	}
+
+	handler := &LocalAdapter{
+		cfg:    cfg,
+		client: client,
+		rule:   rule,
+		worker: worker,
+		cvt:    cvt,
+	}
+
+	return handler, nil
+}
+
+func (adp *LocalAdapter) Convert(ctx context.Context, source string) error {
+	target, err := adp.rule.Map(source)
+	if err != nil {
+		if errors.Is(err, errdefs.ErrAlreadyConverted) {
+			logrus.Infof("image has been converted: %s", source)
+			return nil
+		}
+		return errors.Wrap(err, "create target reference by rule")
+	}
+
+	ctx, done, err := adp.client.WithLease(ctx)
+	if err != nil {
+		return errors.Wrap(err, "create lease")
+	}
+	defer done(ctx)
+
+	return adp.cvt.Convert(ctx, source, target)
+}
+
+func (adp *LocalAdapter) Dispatch(ctx context.Context, ref string, sync bool) error {
+	taskID := task.Manager.Create(ref)
+
+	if sync {
+		// FIXME: The synchronous conversion task should also be
+		// executed in a limited worker queue.
+		return metrics.Conversion.OpWrap(func() error {
+			err := adp.Convert(ctx, ref)
+			task.Manager.Finish(taskID, err)
+			return err
+		}, "convert")
+	}
+
+	adp.worker.Dispatch(func() error {
+		return metrics.Conversion.OpWrap(func() error {
+			err := adp.Convert(context.Background(), ref)
+			task.Manager.Finish(taskID, err)
+			return err
+		}, "convert")
+	})
+
+	return nil
+}
+
+func (adp *LocalAdapter) CheckHealth(ctx context.Context) error {
+	health, err := adp.client.IsServing(ctx)
+
+	msg := "containerd service is unhealthy"
+	if err != nil {
+		return errors.Wrap(err, msg)
+	}
+
+	if !health {
+		return fmt.Errorf(msg)
+	}
+
+	return nil
+}

pkg/converter/rule.go → pkg/adapter/rule.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package converter
+package adapter
 
 import (
 	"errors"

pkg/converter/worker.go → pkg/adapter/worker.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package converter
+package adapter
 
 import (
 	"errors"

pkg/config/config.go

@@ -15,8 +15,13 @@
 package config
 
 import (
+	"encoding/base64"
+	"fmt"
 	"io/ioutil"
+	"net/url"
+	"strings"
 
+	"github.com/goharbor/acceleration-service/pkg/remote"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
 )
@@ -89,3 +94,45 @@ func Parse(configPath string) (*Config, error) {
 
 	return &config, nil
 }
+
+func (cfg *Config) Host(ref string) (remote.CredentialFunc, bool, error) {
+	authorizer := func(ref string) (*SourceConfig, error) {
+		refURL, err := url.Parse(fmt.Sprintf("dummy://%s", ref))
+		if err != nil {
+			return nil, errors.Wrap(err, "parse reference of source image")
+		}
+
+		auth, ok := cfg.Provider.Source[refURL.Host]
+		if !ok {
+			return nil, fmt.Errorf("not found matched hostname %s in config", refURL.Host)
+		}
+
+		return &auth, nil
+	}
+
+	auth, err := authorizer(ref)
+	if err != nil {
+		return nil, false, err
+	}
+
+	return func(host string) (string, string, error) {
+		auth, err := authorizer(host)
+		if err != nil {
+			return "", "", err
+		}
+
+		// Leave auth empty if no authorization be required
+		if strings.TrimSpace(auth.Auth) == "" {
+			return "", "", nil
+		}
+		decoded, err := base64.StdEncoding.DecodeString(auth.Auth)
+		if err != nil {
+			return "", "", errors.Wrap(err, "decode base64 encoded auth string")
+		}
+		ary := strings.Split(string(decoded), ":")
+		if len(ary) != 2 {
+			return "", "", errors.New("invalid base64 encoded auth string")
+		}
+		return ary[0], ary[1], nil
+	}, auth.Insecure, nil
+}