feat: extend the p2p preheat policy

Add the field extra_attrs to the p2p preheat policy for the provider to
define their specified parameters when preheating.

Signed-off-by: chlins <chlins.zhang@gmail.com>

Makefile

@@ -339,17 +339,17 @@ check_environment:
 compile_core: gen_apis
 	@echo "compiling binary for core (golang image)..."
 	@echo $(GOBUILDPATHINCONTAINER)
-	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_CORE) $(GOBUILDIMAGE) $(GOIMAGEBUILD_CORE) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_CORE)/$(CORE_BINARYNAME)
+	@$(DOCKERCMD) run -e GOARCH=amd64 --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_CORE) $(GOBUILDIMAGE) $(GOIMAGEBUILD_CORE) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_CORE)/$(CORE_BINARYNAME)
 	@echo "Done."
 
 compile_jobservice:
 	@echo "compiling binary for jobservice (golang image)..."
-	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_JOBSERVICE) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_JOBSERVICE)/$(JOBSERVICEBINARYNAME)
+	@$(DOCKERCMD) run -e GOARCH=amd64 --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_JOBSERVICE) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_JOBSERVICE)/$(JOBSERVICEBINARYNAME)
 	@echo "Done."
 
 compile_registryctl:
 	@echo "compiling binary for harbor registry controller (golang image)..."
-	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_REGISTRYCTL) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_REGISTRYCTL)/$(REGISTRYCTLBINARYNAME)
+	@$(DOCKERCMD) run -e GOARCH=amd64 --rm -v $(BUILDPATH):$(GOBUILDPATHINCONTAINER) -w $(GOBUILDPATH_REGISTRYCTL) $(GOBUILDIMAGE) $(GOIMAGEBUILD_COMMON) -o $(GOBUILDPATHINCONTAINER)/$(GOBUILDMAKEPATH_REGISTRYCTL)/$(REGISTRYCTLBINARYNAME)
 	@echo "Done."
 
 compile_standalone_db_migrator:
@@ -468,7 +468,7 @@ misspell:
 	@echo checking misspell...
 	@find . -type d \( -path ./tests \) -prune -o -name '*.go' -print | xargs misspell -error
 
-# golangci-lint binary installation or refer to https://golangci-lint.run/usage/install/#local-installation 
+# golangci-lint binary installation or refer to https://golangci-lint.run/usage/install/#local-installation
 # curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.55.2
 GOLANGCI_LINT := $(shell go env GOPATH)/bin/golangci-lint
 lint:

api/v2.0/swagger.yaml

@@ -1193,7 +1193,7 @@ paths:
         '404':
           $ref: '#/responses/404'
         '422':
-          $ref: '#/responses/422'  
+          $ref: '#/responses/422'
         '500':
           $ref: '#/responses/500'
   /projects/{project_name}/repositories/{repository_name}/artifacts/{reference}/scan/stop:
@@ -1226,7 +1226,7 @@ paths:
         '404':
           $ref: '#/responses/404'
         '422':
-          $ref: '#/responses/422'  
+          $ref: '#/responses/422'
         '500':
           $ref: '#/responses/500'
   /projects/{project_name}/repositories/{repository_name}/artifacts/{reference}/scan/{report_id}/log:
@@ -6553,7 +6553,7 @@ responses:
         description: The ID of the corresponding request for the response
         type: string
     schema:
-      $ref: '#/definitions/Errors'    
+      $ref: '#/definitions/Errors'
   '500':
     description: Internal server error
     headers:
@@ -7095,9 +7095,9 @@ definitions:
         type: boolean
         description: Whether the preheat policy enabled
         x-omitempty: false
-      scope:
+      extra_attrs:
         type: string
-        description: The scope of preheat policy
+        description: The extra attributes of preheat policy
       creation_time:
         type: string
         format: date-time
@@ -7937,7 +7937,7 @@ definitions:
     properties:
       resource:
         type: string
-        description: The resource of the access. Possible resources are listed here for system and project level https://github.com/goharbor/harbor/blob/main/src/common/rbac/const.go 
+        description: The resource of the access. Possible resources are listed here for system and project level https://github.com/goharbor/harbor/blob/main/src/common/rbac/const.go
       action:
         type: string
         description: The action of the access. Possible actions are *, pull, push, create, read, update, delete, list, operate, scanner-pull and stop.
@@ -10112,4 +10112,4 @@ definitions:
       scan_type:
         type: string
         description: 'The scan type for the scan request. Two options are currently supported, vulnerability and sbom'
-        enum: [ vulnerability, sbom ]
+        enum: [ vulnerability, sbom ]

make/migrations/postgresql/0160_2.13.0_schema.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE p2p_preheat_policy DROP COLUMN IF EXISTS scope;
+ALTER TABLE p2p_preheat_policy ADD COLUMN IF NOT EXISTS extra_attrs text;

make/photon/Makefile

@@ -232,8 +232,8 @@ define _build_base
 		fi ; \
 	fi
 endef
-
-build: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_trivy_adapter _build_redis _compile_and_build_exporter
+build: _build_core _build_jobservice
+build1: _build_prepare _build_db _build_portal _build_core _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_trivy_adapter _build_redis _compile_and_build_exporter
 	@if [ -n "$(REGISTRYUSER)"  ] && [ -n "$(REGISTRYPASSWORD)" ] ; then \
 		docker logout ; \
 	fi
@@ -246,4 +246,3 @@ cleanimage:
 
 .PHONY: clean
 clean: cleanimage
-

src/controller/p2p/preheat/enforcer.go

@@ -402,7 +402,7 @@ func (de *defaultEnforcer) launchExecutions(ctx context.Context, candidates []*s
 	// Start tasks
 	count := 0
 	for _, c := range candidates {
-		if _, err = de.startTask(ctx, eid, c, insData, pl.Scope); err != nil {
+		if _, err = de.startTask(ctx, eid, c, insData, pl.ExtraAttrs); err != nil {
 			// Just log the error and skip
 			log.Errorf("start task error for preheating image: %s/%s:%s@%s", c.Namespace, c.Repository, c.Tags[0], c.Digest)
 			continue
@@ -421,7 +421,7 @@ func (de *defaultEnforcer) launchExecutions(ctx context.Context, candidates []*s
 }
 
 // startTask starts the preheat task(job) for the given candidate
-func (de *defaultEnforcer) startTask(ctx context.Context, executionID int64, candidate *selector.Candidate, instance, scope string) (int64, error) {
+func (de *defaultEnforcer) startTask(ctx context.Context, executionID int64, candidate *selector.Candidate, instance string, extraAttrs map[string]interface{}) (int64, error) {
 	u, err := de.fullURLGetter(candidate)
 	if err != nil {
 		return -1, err
@@ -438,10 +438,10 @@ func (de *defaultEnforcer) startTask(ctx context.Context, executionID int64, can
 		Headers: map[string]interface{}{
 			accessCredHeaderKey: cred,
 		},
-		ImageName: fmt.Sprintf("%s/%s", candidate.Namespace, candidate.Repository),
-		Tag:       candidate.Tags[0],
-		Digest:    candidate.Digest,
-		Scope:     scope,
+		ImageName:  fmt.Sprintf("%s/%s", candidate.Namespace, candidate.Repository),
+		Tag:        candidate.Tags[0],
+		Digest:     candidate.Digest,
+		ExtraAttrs: extraAttrs,
 	}
 
 	piData, err := pi.ToJSON()

src/controller/p2p/preheat/enforcer_test.go

@@ -210,7 +210,6 @@ func mockPolicies() []*po.Schema {
 				Type: po.TriggerTypeManual,
 			},
 			Enabled:     true,
-			Scope:       "single_peer",
 			CreatedAt:   time.Now().UTC(),
 			UpdatedTime: time.Now().UTC(),
 		}, {
@@ -236,7 +235,6 @@ func mockPolicies() []*po.Schema {
 			Trigger: &po.Trigger{
 				Type: po.TriggerTypeEventBased,
 			},
-			Scope:       "all_peers",
 			Enabled:     true,
 			CreatedAt:   time.Now().UTC(),
 			UpdatedTime: time.Now().UTC(),

src/go.mod

@@ -197,8 +197,8 @@ require (
 replace (
 	github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.0+incompatible
 	github.com/docker/distribution => github.com/distribution/distribution v2.8.2+incompatible
+	github.com/gocraft/work => github.com/goharbor/work v0.5.1-patch
 	github.com/goharbor/harbor => ../
 	github.com/gomodule/redigo => github.com/gomodule/redigo v1.8.8
 	google.golang.org/api => google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff
-	github.com/gocraft/work => github.com/goharbor/work v0.5.1-patch
 )

src/go.sum

@@ -195,8 +195,6 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/gocarina/gocsv v0.0.0-20210516172204-ca9e8a8ddea8 h1:hp1oqdzmv37vPLYFGjuM/RmUgUMfD9vQfMszc54l55Y=
 github.com/gocarina/gocsv v0.0.0-20210516172204-ca9e8a8ddea8/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
-github.com/gocraft/work v0.5.1 h1:3bRjMiOo6N4zcRgZWV3Y7uX7R22SF+A9bPTk4xRXr34=
-github.com/gocraft/work v0.5.1/go.mod h1:pc3n9Pb5FAESPPGfM0nL+7Q1xtgtRnF8rr/azzhQVlM=
 github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=

src/pkg/p2p/preheat/models/policy/policy.go

@@ -30,9 +30,6 @@ func init() {
 	beego_orm.RegisterModel(&Schema{})
 }
 
-// ScopeType represents the preheat scope type.
-type ScopeType = string
-
 const (
 	// Filters:
 	//  Repository : type=Repository value=name text (double star pattern used)
@@ -58,11 +55,6 @@ const (
 	TriggerTypeScheduled TriggerType = "scheduled"
 	// TriggerTypeEventBased represents the event_based trigger type
 	TriggerTypeEventBased TriggerType = "event_based"
-
-	// ScopeTypeSinglePeer represents preheat image to single peer in p2p cluster.
-	ScopeTypeSinglePeer ScopeType = "single_peer"
-	// ScopeTypeAllPeers represents preheat image to all peers in p2p cluster.
-	ScopeTypeAllPeers ScopeType = "all_peers"
 )
 
 // Schema defines p2p preheat policy schema
@@ -80,10 +72,11 @@ type Schema struct {
 	// Use JSON data format (query by trigger type should be supported)
 	TriggerStr string `orm:"column(trigger)" json:"-"`
 	Enabled    bool   `orm:"column(enabled)" json:"enabled"`
-	// Scope decides the preheat scope.
-	Scope       string    `orm:"column(scope)" json:"scope"`
-	CreatedAt   time.Time `orm:"column(creation_time)" json:"creation_time"`
-	UpdatedTime time.Time `orm:"column(update_time)" json:"update_time"`
+	// ExtraAttrs is used to store extra attributes provided by vendor.
+	ExtraAttrsStr string                 `orm:"column(extra_attrs)" json:"-"`
+	ExtraAttrs    map[string]interface{} `orm:"-" json:"extra_attrs"`
+	CreatedAt     time.Time              `orm:"column(creation_time)" json:"creation_time"`
+	UpdatedTime   time.Time              `orm:"column(update_time)" json:"update_time"`
 }
 
 // TableName specifies the policy schema table name.
@@ -136,11 +129,6 @@ func (s *Schema) ValidatePreheatPolicy() error {
 		}
 	}
 
-	// validate preheat scope
-	if s.Scope != "" && s.Scope != ScopeTypeSinglePeer && s.Scope != ScopeTypeAllPeers {
-		return errors.New(nil).WithCode(errors.BadRequestCode).WithMessagef("invalid scope for preheat policy: %s", s.Scope)
-	}
-
 	return nil
 }
 
@@ -162,6 +150,14 @@ func (s *Schema) Encode() error {
 		s.TriggerStr = string(triggerStr)
 	}
 
+	if s.ExtraAttrs != nil {
+		extraAttrsStr, err := json.Marshal(s.ExtraAttrs)
+		if err != nil {
+			return err
+		}
+		s.ExtraAttrsStr = string(extraAttrsStr)
+	}
+
 	return nil
 }
 
@@ -181,6 +177,13 @@ func (s *Schema) Decode() error {
 	}
 	s.Trigger = trigger
 
+	// parse extra attributes
+	extraAttrs, err := decodeExtraAttrs(s.ExtraAttrsStr)
+	if err != nil {
+		return err
+	}
+	s.ExtraAttrs = extraAttrs
+
 	return nil
 }
 
@@ -230,3 +233,17 @@ func decodeTrigger(triggerStr string) (*Trigger, error) {
 
 	return trigger, nil
 }
+
+// decodeExtraAttrs parse extraAttrsStr to extraAttrs.
+func decodeExtraAttrs(extraAttrsStr string) (map[string]interface{}, error) {
+	if len(extraAttrsStr) == 0 {
+		return nil, nil
+	}
+
+	extraAttrs := make(map[string]interface{})
+	if err := json.Unmarshal([]byte(extraAttrsStr), &extraAttrs); err != nil {
+		return nil, err
+	}
+
+	return extraAttrs, nil
+}

src/pkg/p2p/preheat/models/policy/policy_test.go

@@ -64,35 +64,28 @@ func (p *PolicyTestSuite) TestValidatePreheatPolicy() {
 	// valid cron string
 	p.schema.Trigger.Settings.Cron = "0 0 0 1 1 *"
 	p.NoError(p.schema.ValidatePreheatPolicy())
-
-	// invalid preheat scope
-	p.schema.Scope = "invalid scope"
-	p.Error(p.schema.ValidatePreheatPolicy())
-	// valid preheat scope
-	p.schema.Scope = "single_peer"
-	p.NoError(p.schema.ValidatePreheatPolicy())
 }
 
 // TestDecode tests decode.
 func (p *PolicyTestSuite) TestDecode() {
 	s := &Schema{
-		ID:          100,
-		Name:        "test-for-decode",
-		Description: "",
-		ProjectID:   1,
-		ProviderID:  1,
-		Filters:     nil,
-		FiltersStr:  "[{\"type\":\"repository\",\"value\":\"**\"},{\"type\":\"tag\",\"value\":\"**\"},{\"type\":\"label\",\"value\":\"test\"}]",
-		Trigger:     nil,
-		TriggerStr:  "{\"type\":\"event_based\",\"trigger_setting\":{\"cron\":\"\"}}",
-		Enabled:     false,
-		Scope:       "all_peers",
+		ID:            100,
+		Name:          "test-for-decode",
+		Description:   "",
+		ProjectID:     1,
+		ProviderID:    1,
+		Filters:       nil,
+		FiltersStr:    "[{\"type\":\"repository\",\"value\":\"**\"},{\"type\":\"tag\",\"value\":\"**\"},{\"type\":\"label\",\"value\":\"test\"}]",
+		Trigger:       nil,
+		TriggerStr:    "{\"type\":\"event_based\",\"trigger_setting\":{\"cron\":\"\"}}",
+		Enabled:       false,
+		ExtraAttrsStr: "{\"key\":\"value\"}",
 	}
 	p.NoError(s.Decode())
 	p.Len(s.Filters, 3)
 	p.NotNil(s.Trigger)
 
-	p.Equal(ScopeTypeAllPeers, s.Scope)
+	p.Equal(map[string]interface{}{"key": "value"}, s.ExtraAttrs)
 
 	// invalid filter or trigger
 	s.FiltersStr = ""
@@ -132,10 +125,12 @@ func (p *PolicyTestSuite) TestEncode() {
 		},
 		TriggerStr: "",
 		Enabled:    false,
-		Scope:      "single_peer",
+		ExtraAttrs: map[string]interface{}{
+			"key": "value",
+		},
 	}
 	p.NoError(s.Encode())
 	p.Equal(`[{"type":"repository","value":"**"},{"type":"tag","value":"**"},{"type":"label","value":"test"}]`, s.FiltersStr)
 	p.Equal(`{"type":"event_based","trigger_setting":{}}`, s.TriggerStr)
-	p.Equal(ScopeTypeSinglePeer, s.Scope)
+	p.Equal(`{"key":"value"}`, s.ExtraAttrsStr)
 }