From 2ed61f8a66783977fa1ae0d7001100add6589485 Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Tue, 27 Jul 2021 12:11:02 +0200 Subject: [PATCH 01/21] parse hcp_packer_registry block --- hcl2template/types.build.go | 96 ++++++++++--------- .../types.build.hcp_packer_registry.go | 38 ++++++++ 2 files changed, 91 insertions(+), 43 deletions(-) create mode 100644 hcl2template/types.build.hcp_packer_registry.go diff --git a/hcl2template/types.build.go b/hcl2template/types.build.go index 4f6557bbde9..3d5f09e05d4 100644 --- a/hcl2template/types.build.go +++ b/hcl2template/types.build.go @@ -22,6 +22,8 @@ const ( buildPostProcessorLabel = "post-processor" buildPostProcessorsLabel = "post-processors" + + buildHCPPackerRegistryLabel = "hcp_packer_registry" ) var buildSchema = &hcl.BodySchema{ @@ -32,6 +34,7 @@ var buildSchema = &hcl.BodySchema{ {Type: buildErrorCleanupProvisionerLabel, LabelNames: []string{"type"}}, {Type: buildPostProcessorLabel, LabelNames: []string{"type"}}, {Type: buildPostProcessorsLabel, LabelNames: []string{}}, + {Type: buildHCPPackerRegistryLabel}, }, } @@ -58,6 +61,8 @@ type BuildBlock struct { // call for example. Description string + HCPPackerRegistry *HCPPackerRegistryBlock + // Sources is the list of sources that we want to start in this build block. Sources []SourceUseBlock @@ -99,49 +104,6 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB build.Name = b.Name build.Description = b.Description - // TODO if hcp_packer_registry block defined create bucket using the config specified otherwise - // load defaults from ENV - // if config has values => override the env. - if env.InPARMode() { - var err error - cfg.bucket, err = packerregistry.NewBucketWithIteration(packerregistry.IterationOptions{ - TemplateBaseDir: cfg.Basedir, - }) - cfg.bucket.LoadDefaultSettingsFromEnv() - if build.Name != "" { - cfg.bucket.Slug = build.Name - } - if err != nil { - diags = append(diags, &hcl.Diagnostic{ - Summary: "Unable to create a valid bucket object for HCP Packer Registry", - Detail: fmt.Sprintf("%s", err), - Severity: hcl.DiagError, - }) - } - } - - for _, buildFrom := range b.FromSources { - ref := sourceRefFromString(buildFrom) - - if ref == NoSource || - !hclsyntax.ValidIdentifier(ref.Type) || - !hclsyntax.ValidIdentifier(ref.Name) { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid " + sourceLabel + " reference", - Detail: "A " + sourceLabel + " type is made of three parts that are" + - "split by a dot `.`; each part must start with a letter and " + - "may contain only letters, digits, underscores, and dashes." + - "A valid source reference looks like: `source.type.name`", Subject: block.DefRange.Ptr(), - }) - continue - } - - // source with no body - build.Sources = append(build.Sources, SourceUseBlock{SourceRef: ref}) - cfg.bucket.RegisterBuildForComponent(ref.String()) - } - body = b.Config content, moreDiags := body.Content(buildSchema) diags = append(diags, moreDiags...) @@ -150,6 +112,13 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB } for _, block := range content.Blocks { switch block.Type { + case buildHCPPackerRegistryLabel: + hcpPackerRegistry, moreDiags := p.decodeHCPRegistry(block) + diags = append(diags, moreDiags...) + if moreDiags.HasErrors() { + continue + } + build.HCPPackerRegistry = hcpPackerRegistry case sourceLabel: ref, moreDiags := p.decodeBuildSource(block) diags = append(diags, moreDiags...) @@ -211,5 +180,46 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB } } + if build.HCPPackerRegistry != nil || env.InPARMode() { + var err error + cfg.bucket, err = packerregistry.NewBucketWithIteration(packerregistry.IterationOptions{ + TemplateBaseDir: cfg.Basedir, + }) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Summary: "Unable to create a valid bucket object for HCP Packer Registry", + Detail: fmt.Sprintf("%s", err), + Severity: hcl.DiagError, + }) + } + cfg.bucket.LoadDefaultSettingsFromEnv() + if build.Name != "" { + cfg.bucket.Slug = build.Name + } + build.HCPPackerRegistry.WriteBucket(cfg.bucket) + } + + for _, buildFrom := range b.FromSources { + ref := sourceRefFromString(buildFrom) + + if ref == NoSource || + !hclsyntax.ValidIdentifier(ref.Type) || + !hclsyntax.ValidIdentifier(ref.Name) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid " + sourceLabel + " reference", + Detail: "A " + sourceLabel + " type is made of three parts that are" + + "split by a dot `.`; each part must start with a letter and " + + "may contain only letters, digits, underscores, and dashes." + + "A valid source reference looks like: `source.type.name`", Subject: block.DefRange.Ptr(), + }) + continue + } + + // source with no body + build.Sources = append(build.Sources, SourceUseBlock{SourceRef: ref}) + cfg.bucket.RegisterBuildForComponent(ref.String()) + } + return build, diags } diff --git a/hcl2template/types.build.hcp_packer_registry.go b/hcl2template/types.build.hcp_packer_registry.go new file mode 100644 index 00000000000..f2aa7c3934d --- /dev/null +++ b/hcl2template/types.build.hcp_packer_registry.go @@ -0,0 +1,38 @@ +package hcl2template + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + packerregistry "github.com/hashicorp/packer/internal/packer_registry" +) + +type HCPPackerRegistryBlock struct { + Description string + + HCL2Ref HCL2Ref +} + +func (b *HCPPackerRegistryBlock) WriteBucket(bucket *packerregistry.Bucket) { + if b == nil { + return + } + bucket.Description = b.Description +} + +func (p *Parser) decodeHCPRegistry(block *hcl.Block) (*HCPPackerRegistryBlock, hcl.Diagnostics) { + par := &HCPPackerRegistryBlock{} + body := block.Body + + var b struct { + Description string `hcl:"description,optional"` + Config hcl.Body `hcl:",remain"` + } + diags := gohcl.DecodeBody(body, nil, &b) + if diags.HasErrors() { + return nil, diags + } + + par.Description = b.Description + + return par, diags +} From 765715f7a80c2680d64fe262063a6e6dd00bdba9 Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Tue, 27 Jul 2021 14:37:38 +0200 Subject: [PATCH 02/21] Add config to hcp_packer_registry block --- hcl2template/types.build.go | 2 +- hcl2template/types.build.hcp_packer_registry.go | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/hcl2template/types.build.go b/hcl2template/types.build.go index 3d5f09e05d4..cbd71772f50 100644 --- a/hcl2template/types.build.go +++ b/hcl2template/types.build.go @@ -196,7 +196,7 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB if build.Name != "" { cfg.bucket.Slug = build.Name } - build.HCPPackerRegistry.WriteBucket(cfg.bucket) + build.HCPPackerRegistry.WriteBucketConfig(cfg.bucket) } for _, buildFrom := range b.FromSources { diff --git a/hcl2template/types.build.hcp_packer_registry.go b/hcl2template/types.build.hcp_packer_registry.go index f2aa7c3934d..5d4fec68992 100644 --- a/hcl2template/types.build.hcp_packer_registry.go +++ b/hcl2template/types.build.hcp_packer_registry.go @@ -8,15 +8,17 @@ import ( type HCPPackerRegistryBlock struct { Description string + Labels map[string]string HCL2Ref HCL2Ref } -func (b *HCPPackerRegistryBlock) WriteBucket(bucket *packerregistry.Bucket) { +func (b *HCPPackerRegistryBlock) WriteBucketConfig(bucket *packerregistry.Bucket) { if b == nil { return } bucket.Description = b.Description + bucket.Labels = b.Labels } func (p *Parser) decodeHCPRegistry(block *hcl.Block) (*HCPPackerRegistryBlock, hcl.Diagnostics) { @@ -24,8 +26,9 @@ func (p *Parser) decodeHCPRegistry(block *hcl.Block) (*HCPPackerRegistryBlock, h body := block.Body var b struct { - Description string `hcl:"description,optional"` - Config hcl.Body `hcl:",remain"` + Description string `hcl:"description,optional"` + Labels map[string]string `hcl:"labels,optional"` + Config hcl.Body `hcl:",remain"` } diags := gohcl.DecodeBody(body, nil, &b) if diags.HasErrors() { @@ -33,6 +36,7 @@ func (p *Parser) decodeHCPRegistry(block *hcl.Block) (*HCPPackerRegistryBlock, h } par.Description = b.Description + par.Labels = b.Labels return par, diags } From 9943301430549d6c76b1cb155eb6715a95b5f178 Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Tue, 27 Jul 2021 14:58:11 +0200 Subject: [PATCH 03/21] get org and proj id from client secrets --- internal/packer_registry/client.go | 57 +++++-------------- internal/packer_registry/env/env.go | 6 -- internal/packer_registry/env/variables.go | 7 +-- internal/packer_registry/registry.go | 28 ++++----- internal/packer_registry/types.bucket.go | 69 +++++++++++++++-------- 5 files changed, 76 insertions(+), 91 deletions(-) diff --git a/internal/packer_registry/client.go b/internal/packer_registry/client.go index 8a6fed60af2..492923a79a3 100644 --- a/internal/packer_registry/client.go +++ b/internal/packer_registry/client.go @@ -1,11 +1,9 @@ package packer_registry import ( - "errors" - packerSvc "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/preview/2021-04-30/client/packer_service" - "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/client/organization_service" - "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/client/project_service" + organizationSvc "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/client/organization_service" + projectSvc "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/client/project_service" "github.com/hashicorp/hcp-sdk-go/httpclient" ) @@ -13,40 +11,21 @@ import ( type ClientConfig struct { ClientID string ClientSecret string - - // OrganizationID is the organization unique identifier on HCP. - OrganizationID string - - // ProjectID is the project unique identifier on HCP. - ProjectID string -} - -func (cfg ClientConfig) Validate() error { - if cfg.OrganizationID == "" { - return &ClientError{ - StatusCode: InvalidClientConfig, - Err: errors.New(`no valid HCP Organization ID found, check that HCP_PACKER_REGISTRY is in the format "HCP_ORG_ID/HCP_PROJ_ID"`), - } - - } - - if cfg.ProjectID == "" { - return &ClientError{ - StatusCode: InvalidClientConfig, - Err: errors.New(`no valid HCP Project ID found, check that HCP_PACKER_REGISTRY is in the format "HCP_ORG_ID/HCP_PROJ_ID"`), - } - } - - return nil } // Client is an HCP client capable of making requests on behalf of a service principal type Client struct { Config ClientConfig - Organization organization_service.ClientService - Project project_service.ClientService + Organization organizationSvc.ClientService + Project projectSvc.ClientService Packer packerSvc.ClientService + + // OrganizationID is the organization unique identifier on HCP. + OrganizationID string + + // ProjectID is the project unique identifier on HCP. + ProjectID string } // NewClient returns an authenticated client to a HCP Packer Registry. @@ -54,13 +33,6 @@ type Client struct { // if not explicitly provided via a valid ClientConfig cfg. // Upon error a HCPClientError will be returned. func NewClient(cfg ClientConfig) (*Client, error) { - if err := cfg.Validate(); err != nil { - return nil, &ClientError{ - StatusCode: InvalidClientConfig, - Err: err, - } - } - cl, err := httpclient.New(httpclient.Config{}) if err != nil { return nil, &ClientError{ @@ -69,10 +41,11 @@ func NewClient(cfg ClientConfig) (*Client, error) { } } - svc := packerSvc.New(cl, nil) return &Client{ - Packer: svc, - Config: cfg, + Packer: packerSvc.New(cl, nil), + Organization: organizationSvc.New(cl, nil), + Project: projectSvc.New(cl, nil), + Config: cfg, }, nil - } + diff --git a/internal/packer_registry/env/env.go b/internal/packer_registry/env/env.go index c67a1dfdd17..8cf22e6ed13 100644 --- a/internal/packer_registry/env/env.go +++ b/internal/packer_registry/env/env.go @@ -12,11 +12,6 @@ func HasClientSecret() bool { return ok } -func HasPackerRegistryDestionation() bool { - _, ok := os.LookupEnv(HCPPackerRegistry) - return ok -} - func HasPackerRegistryBucket() bool { _, ok := os.LookupEnv(HCPPackerBucket) return ok @@ -26,7 +21,6 @@ func InPARMode() bool { checks := []func() bool{ HasClientID, HasClientSecret, - HasPackerRegistryDestionation, } for _, check := range checks { diff --git a/internal/packer_registry/env/variables.go b/internal/packer_registry/env/variables.go index 7fdb2a723fc..569ee345954 100644 --- a/internal/packer_registry/env/variables.go +++ b/internal/packer_registry/env/variables.go @@ -1,8 +1,7 @@ package env const ( - HCPClientID = "HCP_CLIENT_ID" - HCPClientSecret = "HCP_CLIENT_SECRET" - HCPPackerRegistry = "HCP_PACKER_REGISTRY" - HCPPackerBucket = "HCP_PACKER_BUCKET_NAME" + HCPClientID = "HCP_CLIENT_ID" + HCPClientSecret = "HCP_CLIENT_SECRET" + HCPPackerBucket = "HCP_PACKER_BUCKET_NAME" ) diff --git a/internal/packer_registry/registry.go b/internal/packer_registry/registry.go index 9de26707f9e..504b6e7a297 100644 --- a/internal/packer_registry/registry.go +++ b/internal/packer_registry/registry.go @@ -14,8 +14,8 @@ import ( func CreateBucket(ctx context.Context, client *Client, input *models.HashicorpCloudPackerCreateBucketRequest) (string, error) { params := packerSvc.NewCreateBucketParamsWithContext(ctx) - params.LocationOrganizationID = client.Config.OrganizationID - params.LocationProjectID = client.Config.ProjectID + params.LocationOrganizationID = client.OrganizationID + params.LocationProjectID = client.ProjectID params.Body = input resp, err := client.Packer.CreateBucket(params, nil, func(*runtime.ClientOperation) {}) @@ -41,8 +41,8 @@ func UpsertBucket(ctx context.Context, client *Client, input *models.HashicorpCl } params := packerSvc.NewUpdateBucketParamsWithContext(ctx) - params.LocationOrganizationID = client.Config.OrganizationID - params.LocationProjectID = client.Config.ProjectID + params.LocationOrganizationID = client.OrganizationID + params.LocationProjectID = client.ProjectID params.BucketSlug = input.BucketSlug params.Body = &models.HashicorpCloudPackerUpdateBucketRequest{ Description: input.Description, @@ -62,8 +62,8 @@ input: *models.HashicorpCloudPackerCreateIterationRequest{BucketSlug: "bucket na func CreateIteration(ctx context.Context, client *Client, input *models.HashicorpCloudPackerCreateIterationRequest) (string, error) { // Create/find iteration params := packerSvc.NewCreateIterationParamsWithContext(ctx) - params.LocationOrganizationID = client.Config.OrganizationID - params.LocationProjectID = client.Config.ProjectID + params.LocationOrganizationID = client.OrganizationID + params.LocationProjectID = client.ProjectID params.BucketSlug = input.BucketSlug params.Body = input @@ -78,8 +78,8 @@ func CreateIteration(ctx context.Context, client *Client, input *models.Hashicor func GetIteration(ctx context.Context, client *Client, bucketslug string, fingerprint string) (string, error) { // Create/find iteration params := packerSvc.NewGetIterationParamsWithContext(ctx) - params.LocationOrganizationID = client.Config.OrganizationID - params.LocationProjectID = client.Config.ProjectID + params.LocationOrganizationID = client.OrganizationID + params.LocationProjectID = client.ProjectID params.BucketSlug = bucketslug // identifier can be either fingerprint, iterationid, or incremental version // for now, we only care about fingerprint so we're hardcoding it. @@ -95,8 +95,8 @@ func GetIteration(ctx context.Context, client *Client, bucketslug string, finger func CreateBuild(ctx context.Context, client *Client, input *models.HashicorpCloudPackerCreateBuildRequest) (string, error) { params := packerSvc.NewCreateBuildParamsWithContext(ctx) - params.LocationOrganizationID = client.Config.OrganizationID - params.LocationProjectID = client.Config.ProjectID + params.LocationOrganizationID = client.OrganizationID + params.LocationProjectID = client.ProjectID params.BucketSlug = input.BucketSlug params.BuildIterationID = input.Build.IterationID params.Body = input @@ -111,8 +111,8 @@ func CreateBuild(ctx context.Context, client *Client, input *models.HashicorpClo func ListBuilds(ctx context.Context, client *Client, bucketSlug string, iterationID string) ([]*models.HashicorpCloudPackerBuild, error) { params := packerSvc.NewListBuildsParamsWithContext(ctx) - params.LocationOrganizationID = client.Config.OrganizationID - params.LocationProjectID = client.Config.ProjectID + params.LocationOrganizationID = client.OrganizationID + params.LocationProjectID = client.ProjectID params.BucketSlug = bucketSlug params.IterationID = iterationID @@ -127,8 +127,8 @@ func ListBuilds(ctx context.Context, client *Client, bucketSlug string, iteratio func UpdateBuild(ctx context.Context, client *Client, input *models.HashicorpCloudPackerUpdateBuildRequest) (string, error) { params := packerSvc.NewUpdateBuildParamsWithContext(ctx) params.BuildID = input.BuildID - params.LocationOrganizationID = client.Config.OrganizationID - params.LocationProjectID = client.Config.ProjectID + params.LocationOrganizationID = client.OrganizationID + params.LocationProjectID = client.ProjectID params.Body = input resp, err := client.Packer.UpdateBuild(params, nil, func(*runtime.ClientOperation) {}) diff --git a/internal/packer_registry/types.bucket.go b/internal/packer_registry/types.bucket.go index 9c177c32f44..e103eb14f07 100644 --- a/internal/packer_registry/types.bucket.go +++ b/internal/packer_registry/types.bucket.go @@ -6,11 +6,13 @@ import ( "fmt" "log" "os" - "strings" "sync" "github.com/hashicorp/go-multierror" "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/preview/2021-04-30/models" + "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/client/organization_service" + "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/client/project_service" + rmmodels "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/models" "github.com/hashicorp/packer/internal/packer_registry/env" "google.golang.org/grpc/codes" ) @@ -48,10 +50,6 @@ func (b *Bucket) Validate() error { return fmt.Errorf("no Packer bucket name defined; either the environment variable %q is undefined or the HCL configuration has no build name", env.HCPPackerBucket) } - if b.Destination == "" { - return fmt.Errorf("no Packer registry defined; either the environment variable %q is undefined or the HCL configuration has no build name", env.HCPPackerRegistry) - } - return nil } @@ -69,6 +67,15 @@ func (b *Bucket) Initialize(ctx context.Context) error { } } + if err := b.loadOrganizationIDFromCredentials(); err != nil { + return err + } + if err := b.loadProjectIDFromCredentials(); err != nil { + return err + } + + b.Destination = fmt.Sprintf("%s/%s", b.client.OrganizationID, b.client.ProjectID) + bucketInput := &models.HashicorpCloudPackerCreateBucketRequest{ BucketSlug: b.Slug, Description: b.Description, @@ -177,6 +184,38 @@ func (b *Bucket) Initialize(ctx context.Context) error { return errs.ErrorOrNil() } +func (b *Bucket) loadOrganizationIDFromCredentials() error { + // Get the organization ID. + listOrgParams := organization_service.NewOrganizationServiceListParams() + listOrgResp, err := b.client.Organization.OrganizationServiceList(listOrgParams, nil) + if err != nil { + return fmt.Errorf("unable to fetch organization list: %v", err) + } + orgLen := len(listOrgResp.Payload.Organizations) + if orgLen != 1 { + return fmt.Errorf("unexpected number of organizations: expected 1, actual: %v", orgLen) + } + b.client.OrganizationID = listOrgResp.Payload.Organizations[0].ID + return nil +} + +func (b *Bucket) loadProjectIDFromCredentials() error { + // Get the project using the organization ID. + listProjParams := project_service.NewProjectServiceListParams() + listProjParams.ScopeID = &b.client.OrganizationID + scopeType := string(rmmodels.HashicorpCloudResourcemanagerResourceIDResourceTypeORGANIZATION) + listProjParams.ScopeType = &scopeType + listProjResp, err := b.client.Project.ProjectServiceList(listProjParams, nil) + if err != nil { + return fmt.Errorf("unable to fetch project id: %v", err) + } + if len(listProjResp.Payload.Projects) > 1 { + return fmt.Errorf("this version of Packer does not support multiple projects, upgrade to a later provider version and set a project ID on the provider/resources") + } + b.client.ProjectID = listProjResp.Payload.Projects[0].ID + return nil +} + // connect initializes a client connection to a remote HCP Packer Registry service on HCP. // Upon a successful connection the initialized client is persisted on the Bucket b for later usage. func (b *Bucket) connect() error { @@ -344,26 +383,6 @@ func (b *Bucket) LoadDefaultSettingsFromEnv() { b.Slug = os.Getenv(env.HCPPackerBucket) } - loc := os.Getenv(env.HCPPackerRegistry) - locParts := strings.Split(loc, "/") - if len(locParts) != 2 { - // we want an error here. Or at least when we try to create the registry client we fail - return - } - orgID, projID := locParts[0], locParts[1] - - if b.Destination == "" { - b.Destination = loc - } - - if b.Config.OrganizationID == "" { - b.Config.OrganizationID = orgID - } - - if b.Config.ProjectID == "" { - b.Config.ProjectID = projID - } - // Set some iteration values. For Packer RunUUID should always be set. // Creating a bucket differently? Let's not overwrite a UUID that might be set. if b.Iteration.RunUUID == "" { From 92ff0c050c5135ae89fa6f09c3c3c1d82f055abe Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Tue, 27 Jul 2021 15:23:12 +0200 Subject: [PATCH 04/21] move org and proj id logic to client --- internal/packer_registry/client.go | 47 +++++++++++++++++++++++- internal/packer_registry/types.bucket.go | 42 --------------------- 2 files changed, 45 insertions(+), 44 deletions(-) diff --git a/internal/packer_registry/client.go b/internal/packer_registry/client.go index 492923a79a3..1b1987bd4fd 100644 --- a/internal/packer_registry/client.go +++ b/internal/packer_registry/client.go @@ -1,9 +1,12 @@ package packer_registry import ( + "fmt" + packerSvc "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/preview/2021-04-30/client/packer_service" organizationSvc "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/client/organization_service" projectSvc "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/client/project_service" + rmmodels "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/models" "github.com/hashicorp/hcp-sdk-go/httpclient" ) @@ -41,11 +44,51 @@ func NewClient(cfg ClientConfig) (*Client, error) { } } - return &Client{ + client := &Client{ Packer: packerSvc.New(cl, nil), Organization: organizationSvc.New(cl, nil), Project: projectSvc.New(cl, nil), Config: cfg, - }, nil + } + + if err := client.loadOrganizationID(); err != nil { + return nil, err + } + if err := client.loadProjectID(); err != nil { + return nil, err + } + + return client, nil } +func (c *Client) loadOrganizationID() error { + // Get the organization ID. + listOrgParams := organizationSvc.NewOrganizationServiceListParams() + listOrgResp, err := c.Organization.OrganizationServiceList(listOrgParams, nil) + if err != nil { + return fmt.Errorf("unable to fetch organization list: %v", err) + } + orgLen := len(listOrgResp.Payload.Organizations) + if orgLen != 1 { + return fmt.Errorf("unexpected number of organizations: expected 1, actual: %v", orgLen) + } + c.OrganizationID = listOrgResp.Payload.Organizations[0].ID + return nil +} + +func (c *Client) loadProjectID() error { + // Get the project using the organization ID. + listProjParams := projectSvc.NewProjectServiceListParams() + listProjParams.ScopeID = &c.OrganizationID + scopeType := string(rmmodels.HashicorpCloudResourcemanagerResourceIDResourceTypeORGANIZATION) + listProjParams.ScopeType = &scopeType + listProjResp, err := c.Project.ProjectServiceList(listProjParams, nil) + if err != nil { + return fmt.Errorf("unable to fetch project id: %v", err) + } + if len(listProjResp.Payload.Projects) > 1 { + return fmt.Errorf("this version of Packer does not support multiple projects, upgrade to a later provider version and set a project ID on the provider/resources") + } + c.ProjectID = listProjResp.Payload.Projects[0].ID + return nil +} diff --git a/internal/packer_registry/types.bucket.go b/internal/packer_registry/types.bucket.go index e103eb14f07..5c2eb2acdc1 100644 --- a/internal/packer_registry/types.bucket.go +++ b/internal/packer_registry/types.bucket.go @@ -10,9 +10,6 @@ import ( "github.com/hashicorp/go-multierror" "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/preview/2021-04-30/models" - "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/client/organization_service" - "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/client/project_service" - rmmodels "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/models" "github.com/hashicorp/packer/internal/packer_registry/env" "google.golang.org/grpc/codes" ) @@ -67,13 +64,6 @@ func (b *Bucket) Initialize(ctx context.Context) error { } } - if err := b.loadOrganizationIDFromCredentials(); err != nil { - return err - } - if err := b.loadProjectIDFromCredentials(); err != nil { - return err - } - b.Destination = fmt.Sprintf("%s/%s", b.client.OrganizationID, b.client.ProjectID) bucketInput := &models.HashicorpCloudPackerCreateBucketRequest{ @@ -184,38 +174,6 @@ func (b *Bucket) Initialize(ctx context.Context) error { return errs.ErrorOrNil() } -func (b *Bucket) loadOrganizationIDFromCredentials() error { - // Get the organization ID. - listOrgParams := organization_service.NewOrganizationServiceListParams() - listOrgResp, err := b.client.Organization.OrganizationServiceList(listOrgParams, nil) - if err != nil { - return fmt.Errorf("unable to fetch organization list: %v", err) - } - orgLen := len(listOrgResp.Payload.Organizations) - if orgLen != 1 { - return fmt.Errorf("unexpected number of organizations: expected 1, actual: %v", orgLen) - } - b.client.OrganizationID = listOrgResp.Payload.Organizations[0].ID - return nil -} - -func (b *Bucket) loadProjectIDFromCredentials() error { - // Get the project using the organization ID. - listProjParams := project_service.NewProjectServiceListParams() - listProjParams.ScopeID = &b.client.OrganizationID - scopeType := string(rmmodels.HashicorpCloudResourcemanagerResourceIDResourceTypeORGANIZATION) - listProjParams.ScopeType = &scopeType - listProjResp, err := b.client.Project.ProjectServiceList(listProjParams, nil) - if err != nil { - return fmt.Errorf("unable to fetch project id: %v", err) - } - if len(listProjResp.Payload.Projects) > 1 { - return fmt.Errorf("this version of Packer does not support multiple projects, upgrade to a later provider version and set a project ID on the provider/resources") - } - b.client.ProjectID = listProjResp.Payload.Projects[0].ID - return nil -} - // connect initializes a client connection to a remote HCP Packer Registry service on HCP. // Upon a successful connection the initialized client is persisted on the Bucket b for later usage. func (b *Bucket) connect() error { From 9bcd11fbae549babac14bf9482e2d85b31956568 Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Tue, 27 Jul 2021 15:32:40 +0200 Subject: [PATCH 05/21] clean up client config --- internal/packer_registry/client.go | 26 ++++++++++-------------- internal/packer_registry/types.bucket.go | 12 +---------- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/internal/packer_registry/client.go b/internal/packer_registry/client.go index 1b1987bd4fd..75943ccde8e 100644 --- a/internal/packer_registry/client.go +++ b/internal/packer_registry/client.go @@ -10,19 +10,11 @@ import ( "github.com/hashicorp/hcp-sdk-go/httpclient" ) -// ClientConfig specifies configuration for the client that interacts with HCP -type ClientConfig struct { - ClientID string - ClientSecret string -} - // Client is an HCP client capable of making requests on behalf of a service principal type Client struct { - Config ClientConfig - + Packer packerSvc.ClientService Organization organizationSvc.ClientService Project projectSvc.ClientService - Packer packerSvc.ClientService // OrganizationID is the organization unique identifier on HCP. OrganizationID string @@ -32,10 +24,9 @@ type Client struct { } // NewClient returns an authenticated client to a HCP Packer Registry. -// Client authentication requires the following environment variables be set HCP_CLIENT_ID, HCP_CLIENT_SECRET, and HCP_PACKER_REGISTRY. -// if not explicitly provided via a valid ClientConfig cfg. +// Client authentication requires the following environment variables be set HCP_CLIENT_ID and HCP_CLIENT_SECRET. // Upon error a HCPClientError will be returned. -func NewClient(cfg ClientConfig) (*Client, error) { +func NewClient() (*Client, error) { cl, err := httpclient.New(httpclient.Config{}) if err != nil { return nil, &ClientError{ @@ -48,14 +39,19 @@ func NewClient(cfg ClientConfig) (*Client, error) { Packer: packerSvc.New(cl, nil), Organization: organizationSvc.New(cl, nil), Project: projectSvc.New(cl, nil), - Config: cfg, } if err := client.loadOrganizationID(); err != nil { - return nil, err + return nil, &ClientError{ + StatusCode: InvalidClientConfig, + Err: err, + } } if err := client.loadProjectID(); err != nil { - return nil, err + return nil, &ClientError{ + StatusCode: InvalidClientConfig, + Err: err, + } } return client, nil diff --git a/internal/packer_registry/types.bucket.go b/internal/packer_registry/types.bucket.go index 5c2eb2acdc1..aa520c14b57 100644 --- a/internal/packer_registry/types.bucket.go +++ b/internal/packer_registry/types.bucket.go @@ -19,7 +19,6 @@ type Bucket struct { Description string Destination string Labels map[string]string - Config ClientConfig *Iteration client *Client } @@ -46,7 +45,6 @@ func (b *Bucket) Validate() error { if b.Slug == "" { return fmt.Errorf("no Packer bucket name defined; either the environment variable %q is undefined or the HCL configuration has no build name", env.HCPPackerBucket) } - return nil } @@ -177,7 +175,7 @@ func (b *Bucket) Initialize(ctx context.Context) error { // connect initializes a client connection to a remote HCP Packer Registry service on HCP. // Upon a successful connection the initialized client is persisted on the Bucket b for later usage. func (b *Bucket) connect() error { - registryClient, err := NewClient(b.Config) + registryClient, err := NewClient() if err != nil { return errors.New("Failed to create client connection to artifact registry: " + err.Error()) } @@ -328,14 +326,6 @@ func (b *Bucket) AddBuildMetadata(name string, data map[string]string) error { // Load defaults from environment variables func (b *Bucket) LoadDefaultSettingsFromEnv() { - if b.Config.ClientID == "" { - b.Config.ClientID = os.Getenv(env.HCPClientID) - } - - if b.Config.ClientSecret == "" { - b.Config.ClientSecret = os.Getenv(env.HCPClientSecret) - } - // Configure HCP registry destination if b.Slug == "" { b.Slug = os.Getenv(env.HCPPackerBucket) From da7b756f2885a5ddcb30e06d41c817f4fcce073b Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Tue, 27 Jul 2021 15:36:42 +0200 Subject: [PATCH 06/21] rename InPARMode to HasHCPCredentials --- command/core_wrapper.go | 2 +- hcl2template/types.build.go | 2 +- internal/packer_registry/env/env.go | 2 +- packer/core.go | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/command/core_wrapper.go b/command/core_wrapper.go index 30f935bff5e..f06ef99ef10 100644 --- a/command/core_wrapper.go +++ b/command/core_wrapper.go @@ -42,7 +42,7 @@ func (c *CoreWrapper) PluginRequirements() (plugingetter.Requirements, hcl.Diagn func (c *CoreWrapper) ConfiguredArtifactMetadataPublisher() (*packerregistry.Bucket, hcl.Diagnostics) { // JSON can only configure its Bucket through Env. variables so if not in PAR mode // we don't really care if bucket is nil or set to a bunch of zero values. - if !env.InPARMode() { + if !env.HasHCPCredentials() { return nil, hcl.Diagnostics{ &hcl.Diagnostic{ Summary: "Publishing build artifacts to HCP Packer Registry not enabled", diff --git a/hcl2template/types.build.go b/hcl2template/types.build.go index cbd71772f50..49595d5542b 100644 --- a/hcl2template/types.build.go +++ b/hcl2template/types.build.go @@ -180,7 +180,7 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB } } - if build.HCPPackerRegistry != nil || env.InPARMode() { + if build.HCPPackerRegistry != nil && env.HasHCPCredentials() { var err error cfg.bucket, err = packerregistry.NewBucketWithIteration(packerregistry.IterationOptions{ TemplateBaseDir: cfg.Basedir, diff --git a/internal/packer_registry/env/env.go b/internal/packer_registry/env/env.go index 8cf22e6ed13..a671af3a7d0 100644 --- a/internal/packer_registry/env/env.go +++ b/internal/packer_registry/env/env.go @@ -17,7 +17,7 @@ func HasPackerRegistryBucket() bool { return ok } -func InPARMode() bool { +func HasHCPCredentials() bool { checks := []func() bool{ HasClientID, HasClientSecret, diff --git a/packer/core.go b/packer/core.go index 38777a1d5d6..22a2e28514b 100644 --- a/packer/core.go +++ b/packer/core.go @@ -142,7 +142,7 @@ func (core *Core) Initialize() error { packersdk.LogSecretFilter.Set(secret) } - if env.InPARMode() { + if env.HasHCPCredentials() { var err error core.bucket, err = packerregistry.NewBucketWithIteration(packerregistry.IterationOptions{ TemplateBaseDir: filepath.Dir(core.Template.Path), @@ -390,7 +390,7 @@ func (c *Core) Build(n string) (packersdk.Build, error) { "post-processor type not found: %s", rawP.Type) } - if env.InPARMode() && c.bucket != nil { + if env.HasHCPCredentials() && c.bucket != nil { postProcessor = &RegistryPostProcessor{ BuilderType: n, ArtifactMetadataPublisher: c.bucket, @@ -414,7 +414,7 @@ func (c *Core) Build(n string) (packersdk.Build, error) { postProcessors = append(postProcessors, current) } - if env.InPARMode() && c.bucket != nil { + if env.HasHCPCredentials() && c.bucket != nil { postProcessors = append(postProcessors, []CoreBuildPostProcessor{ { PostProcessor: &RegistryPostProcessor{ @@ -427,7 +427,7 @@ func (c *Core) Build(n string) (packersdk.Build, error) { // TODO hooks one day - if env.InPARMode() && c.bucket != nil { + if env.HasHCPCredentials() && c.bucket != nil { builder = &RegistryBuilder{ Name: n, ArtifactMetadataPublisher: c.bucket, From b603f987b145ca043b1c1eb2e22b5154a786ee58 Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Tue, 27 Jul 2021 16:23:48 +0200 Subject: [PATCH 07/21] add HCP_PACKER_REGISTRY back to enable PAR when hcp block not present --- command/core_wrapper.go | 2 +- hcl2template/types.build.go | 5 ++--- hcl2template/types.build.hcp_packer_registry.go | 3 --- internal/packer_registry/env/env.go | 5 +++++ internal/packer_registry/env/variables.go | 7 ++++--- packer/core.go | 8 ++++---- 6 files changed, 16 insertions(+), 14 deletions(-) diff --git a/command/core_wrapper.go b/command/core_wrapper.go index f06ef99ef10..c3d824d4b97 100644 --- a/command/core_wrapper.go +++ b/command/core_wrapper.go @@ -42,7 +42,7 @@ func (c *CoreWrapper) PluginRequirements() (plugingetter.Requirements, hcl.Diagn func (c *CoreWrapper) ConfiguredArtifactMetadataPublisher() (*packerregistry.Bucket, hcl.Diagnostics) { // JSON can only configure its Bucket through Env. variables so if not in PAR mode // we don't really care if bucket is nil or set to a bunch of zero values. - if !env.HasHCPCredentials() { + if !env.IsPAREnabled() { return nil, hcl.Diagnostics{ &hcl.Diagnostic{ Summary: "Publishing build artifacts to HCP Packer Registry not enabled", diff --git a/hcl2template/types.build.go b/hcl2template/types.build.go index 49595d5542b..1a24b43a0c2 100644 --- a/hcl2template/types.build.go +++ b/hcl2template/types.build.go @@ -7,7 +7,6 @@ import ( "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclsyntax" packerregistry "github.com/hashicorp/packer/internal/packer_registry" - "github.com/hashicorp/packer/internal/packer_registry/env" ) const ( @@ -180,7 +179,7 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB } } - if build.HCPPackerRegistry != nil && env.HasHCPCredentials() { + if build.HCPPackerRegistry != nil { var err error cfg.bucket, err = packerregistry.NewBucketWithIteration(packerregistry.IterationOptions{ TemplateBaseDir: cfg.Basedir, @@ -193,7 +192,7 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB }) } cfg.bucket.LoadDefaultSettingsFromEnv() - if build.Name != "" { + if cfg.bucket.Slug == "" && build.Name != "" { cfg.bucket.Slug = build.Name } build.HCPPackerRegistry.WriteBucketConfig(cfg.bucket) diff --git a/hcl2template/types.build.hcp_packer_registry.go b/hcl2template/types.build.hcp_packer_registry.go index 5d4fec68992..35b9a8601c5 100644 --- a/hcl2template/types.build.hcp_packer_registry.go +++ b/hcl2template/types.build.hcp_packer_registry.go @@ -14,9 +14,6 @@ type HCPPackerRegistryBlock struct { } func (b *HCPPackerRegistryBlock) WriteBucketConfig(bucket *packerregistry.Bucket) { - if b == nil { - return - } bucket.Description = b.Description bucket.Labels = b.Labels } diff --git a/internal/packer_registry/env/env.go b/internal/packer_registry/env/env.go index a671af3a7d0..d86de100635 100644 --- a/internal/packer_registry/env/env.go +++ b/internal/packer_registry/env/env.go @@ -31,3 +31,8 @@ func HasHCPCredentials() bool { return true } + +func IsPAREnabled() bool { + _, ok := os.LookupEnv(HCPPackerRegistry) + return ok +} diff --git a/internal/packer_registry/env/variables.go b/internal/packer_registry/env/variables.go index 569ee345954..7fdb2a723fc 100644 --- a/internal/packer_registry/env/variables.go +++ b/internal/packer_registry/env/variables.go @@ -1,7 +1,8 @@ package env const ( - HCPClientID = "HCP_CLIENT_ID" - HCPClientSecret = "HCP_CLIENT_SECRET" - HCPPackerBucket = "HCP_PACKER_BUCKET_NAME" + HCPClientID = "HCP_CLIENT_ID" + HCPClientSecret = "HCP_CLIENT_SECRET" + HCPPackerRegistry = "HCP_PACKER_REGISTRY" + HCPPackerBucket = "HCP_PACKER_BUCKET_NAME" ) diff --git a/packer/core.go b/packer/core.go index 22a2e28514b..9080299e501 100644 --- a/packer/core.go +++ b/packer/core.go @@ -142,7 +142,7 @@ func (core *Core) Initialize() error { packersdk.LogSecretFilter.Set(secret) } - if env.HasHCPCredentials() { + if env.IsPAREnabled() { var err error core.bucket, err = packerregistry.NewBucketWithIteration(packerregistry.IterationOptions{ TemplateBaseDir: filepath.Dir(core.Template.Path), @@ -390,7 +390,7 @@ func (c *Core) Build(n string) (packersdk.Build, error) { "post-processor type not found: %s", rawP.Type) } - if env.HasHCPCredentials() && c.bucket != nil { + if env.IsPAREnabled() && c.bucket != nil { postProcessor = &RegistryPostProcessor{ BuilderType: n, ArtifactMetadataPublisher: c.bucket, @@ -414,7 +414,7 @@ func (c *Core) Build(n string) (packersdk.Build, error) { postProcessors = append(postProcessors, current) } - if env.HasHCPCredentials() && c.bucket != nil { + if env.IsPAREnabled() && c.bucket != nil { postProcessors = append(postProcessors, []CoreBuildPostProcessor{ { PostProcessor: &RegistryPostProcessor{ @@ -427,7 +427,7 @@ func (c *Core) Build(n string) (packersdk.Build, error) { // TODO hooks one day - if env.HasHCPCredentials() && c.bucket != nil { + if env.IsPAREnabled() && c.bucket != nil { builder = &RegistryBuilder{ Name: n, ArtifactMetadataPublisher: c.bucket, From 7e8756ed1a1d785a462341e3a915af2433be94b7 Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Tue, 27 Jul 2021 16:33:28 +0200 Subject: [PATCH 08/21] validate credentials when creating client --- internal/packer_registry/client.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/packer_registry/client.go b/internal/packer_registry/client.go index 75943ccde8e..6a4d9a12713 100644 --- a/internal/packer_registry/client.go +++ b/internal/packer_registry/client.go @@ -8,6 +8,7 @@ import ( projectSvc "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/client/project_service" rmmodels "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/models" "github.com/hashicorp/hcp-sdk-go/httpclient" + "github.com/hashicorp/packer/internal/packer_registry/env" ) // Client is an HCP client capable of making requests on behalf of a service principal @@ -27,6 +28,13 @@ type Client struct { // Client authentication requires the following environment variables be set HCP_CLIENT_ID and HCP_CLIENT_SECRET. // Upon error a HCPClientError will be returned. func NewClient() (*Client, error) { + if !env.HasHCPCredentials() { + return nil, &ClientError{ + StatusCode: InvalidClientConfig, + Err: fmt.Errorf("the client authentication requires both HCP_CLIENT_ID and HCP_CLIENT_SECRET environment variables to be set"), + } + } + cl, err := httpclient.New(httpclient.Config{}) if err != nil { return nil, &ClientError{ From 3cc3a7339895ae3a0906a6c58ef3f22e1e9cbf84 Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Tue, 27 Jul 2021 16:41:49 +0200 Subject: [PATCH 09/21] fix - allow hcl2 config par enable without block --- hcl2template/types.build.go | 3 ++- hcl2template/types.build.hcp_packer_registry.go | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hcl2template/types.build.go b/hcl2template/types.build.go index 1a24b43a0c2..b9aa1156c70 100644 --- a/hcl2template/types.build.go +++ b/hcl2template/types.build.go @@ -7,6 +7,7 @@ import ( "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclsyntax" packerregistry "github.com/hashicorp/packer/internal/packer_registry" + "github.com/hashicorp/packer/internal/packer_registry/env" ) const ( @@ -179,7 +180,7 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB } } - if build.HCPPackerRegistry != nil { + if build.HCPPackerRegistry != nil || env.IsPAREnabled() { var err error cfg.bucket, err = packerregistry.NewBucketWithIteration(packerregistry.IterationOptions{ TemplateBaseDir: cfg.Basedir, diff --git a/hcl2template/types.build.hcp_packer_registry.go b/hcl2template/types.build.hcp_packer_registry.go index 35b9a8601c5..5d4fec68992 100644 --- a/hcl2template/types.build.hcp_packer_registry.go +++ b/hcl2template/types.build.hcp_packer_registry.go @@ -14,6 +14,9 @@ type HCPPackerRegistryBlock struct { } func (b *HCPPackerRegistryBlock) WriteBucketConfig(bucket *packerregistry.Bucket) { + if b == nil { + return + } bucket.Description = b.Description bucket.Labels = b.Labels } From 31f9410adbffb7ec29eed04ca399f17a82a57949 Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Tue, 27 Jul 2021 16:59:37 +0200 Subject: [PATCH 10/21] fix build registration --- .../build/hcp_packer_registry.pkr.hcl | 18 ++++++++ hcl2template/types.build.go | 44 ++++++++++--------- hcl2template/types.build_test.go | 35 +++++++++++++++ 3 files changed, 77 insertions(+), 20 deletions(-) create mode 100644 hcl2template/testdata/build/hcp_packer_registry.pkr.hcl diff --git a/hcl2template/testdata/build/hcp_packer_registry.pkr.hcl b/hcl2template/testdata/build/hcp_packer_registry.pkr.hcl new file mode 100644 index 00000000000..59339558606 --- /dev/null +++ b/hcl2template/testdata/build/hcp_packer_registry.pkr.hcl @@ -0,0 +1,18 @@ +// starts resources to provision them. +build { + hcp_packer_registry { + description = < Date: Tue, 27 Jul 2021 17:23:24 +0200 Subject: [PATCH 11/21] update error msg --- internal/packer_registry/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/packer_registry/client.go b/internal/packer_registry/client.go index 6a4d9a12713..cad10f0efd9 100644 --- a/internal/packer_registry/client.go +++ b/internal/packer_registry/client.go @@ -91,7 +91,7 @@ func (c *Client) loadProjectID() error { return fmt.Errorf("unable to fetch project id: %v", err) } if len(listProjResp.Payload.Projects) > 1 { - return fmt.Errorf("this version of Packer does not support multiple projects, upgrade to a later provider version and set a project ID on the provider/resources") + return fmt.Errorf("this version of Packer does not support multiple projects") } c.ProjectID = listProjResp.Payload.Projects[0].ID return nil From 0110f3ab9b4068643e4466e69bf5c6d73f9f0d72 Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Wed, 28 Jul 2021 11:03:03 +0200 Subject: [PATCH 12/21] fix IsPAREnabled logic --- internal/packer_registry/env/env.go | 9 ++-- internal/packer_registry/env/env_test.go | 53 ++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 internal/packer_registry/env/env_test.go diff --git a/internal/packer_registry/env/env.go b/internal/packer_registry/env/env.go index d86de100635..9f8f4961256 100644 --- a/internal/packer_registry/env/env.go +++ b/internal/packer_registry/env/env.go @@ -1,6 +1,9 @@ package env -import "os" +import ( + "os" + "strings" +) func HasClientID() bool { _, ok := os.LookupEnv(HCPClientID) @@ -33,6 +36,6 @@ func HasHCPCredentials() bool { } func IsPAREnabled() bool { - _, ok := os.LookupEnv(HCPPackerRegistry) - return ok + val, ok := os.LookupEnv(HCPPackerRegistry) + return ok && strings.ToLower(val) != "off" && val != "0" } diff --git a/internal/packer_registry/env/env_test.go b/internal/packer_registry/env/env_test.go new file mode 100644 index 00000000000..5d3a9afa438 --- /dev/null +++ b/internal/packer_registry/env/env_test.go @@ -0,0 +1,53 @@ +package env + +import ( + "os" + "testing" +) + +func Test_IsPAREnabled(t *testing.T) { + tcs := []struct { + name string + value string + output bool + }{ + { + name: "set with 1", + value: "1", + output: true, + }, + { + name: "set with ON", + value: "ON", + output: true, + }, + { + name: "set with 0", + value: "0", + output: false, + }, + { + name: "set with OFF", + value: "OFF", + output: false, + }, + { + name: "unset", + value: "", + output: false, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + if tc.value != "" { + _ = os.Setenv(HCPPackerRegistry, tc.value) + defer os.Unsetenv(HCPPackerRegistry) + } + out := IsPAREnabled() + if out != tc.output { + t.Fatalf("unexpected output: %t", out) + } + }) + } +} From f6749a20e97c01d3795d0e82d6556ac64e8fded0 Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Wed, 28 Jul 2021 11:11:41 +0200 Subject: [PATCH 13/21] fix par enabled and bucket validation --- command/core_wrapper.go | 9 ++++----- hcl2template/types.build.go | 2 ++ packer/core.go | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/command/core_wrapper.go b/command/core_wrapper.go index c3d824d4b97..1d0f3290d10 100644 --- a/command/core_wrapper.go +++ b/command/core_wrapper.go @@ -3,7 +3,6 @@ package command import ( "github.com/hashicorp/hcl/v2" packerregistry "github.com/hashicorp/packer/internal/packer_registry" - "github.com/hashicorp/packer/internal/packer_registry/env" "github.com/hashicorp/packer/packer" plugingetter "github.com/hashicorp/packer/packer/plugin-getter" ) @@ -40,9 +39,10 @@ func (c *CoreWrapper) PluginRequirements() (plugingetter.Requirements, hcl.Diagn // ConfiguredArtifactMetadataPublisher returns a configured image bucket that can be used for publishing // build image artifacts to a configured Packer Registry destination. func (c *CoreWrapper) ConfiguredArtifactMetadataPublisher() (*packerregistry.Bucket, hcl.Diagnostics) { - // JSON can only configure its Bucket through Env. variables so if not in PAR mode - // we don't really care if bucket is nil or set to a bunch of zero values. - if !env.IsPAREnabled() { + bucket := c.Core.GetRegistryBucket() + + // If at this point the bucket is nil, it means PAR is not enabled + if bucket == nil { return nil, hcl.Diagnostics{ &hcl.Diagnostic{ Summary: "Publishing build artifacts to HCP Packer Registry not enabled", @@ -53,7 +53,6 @@ func (c *CoreWrapper) ConfiguredArtifactMetadataPublisher() (*packerregistry.Buc } } - bucket := c.Core.GetRegistryBucket() err := bucket.Validate() if err != nil { return nil, hcl.Diagnostics{ diff --git a/hcl2template/types.build.go b/hcl2template/types.build.go index 62964124c56..1e16a88cc8f 100644 --- a/hcl2template/types.build.go +++ b/hcl2template/types.build.go @@ -202,6 +202,8 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildB } } + // Creates a bucket if either a hcp_packer_registry block is set or PAR is enabled via + // environment variable if build.HCPPackerRegistry != nil || env.IsPAREnabled() { var err error cfg.bucket, err = packerregistry.NewBucketWithIteration(packerregistry.IterationOptions{ diff --git a/packer/core.go b/packer/core.go index 9080299e501..87da34ec28b 100644 --- a/packer/core.go +++ b/packer/core.go @@ -390,7 +390,7 @@ func (c *Core) Build(n string) (packersdk.Build, error) { "post-processor type not found: %s", rawP.Type) } - if env.IsPAREnabled() && c.bucket != nil { + if c.bucket != nil { postProcessor = &RegistryPostProcessor{ BuilderType: n, ArtifactMetadataPublisher: c.bucket, @@ -414,7 +414,7 @@ func (c *Core) Build(n string) (packersdk.Build, error) { postProcessors = append(postProcessors, current) } - if env.IsPAREnabled() && c.bucket != nil { + if c.bucket != nil { postProcessors = append(postProcessors, []CoreBuildPostProcessor{ { PostProcessor: &RegistryPostProcessor{ @@ -427,7 +427,7 @@ func (c *Core) Build(n string) (packersdk.Build, error) { // TODO hooks one day - if env.IsPAREnabled() && c.bucket != nil { + if c.bucket != nil { builder = &RegistryBuilder{ Name: n, ArtifactMetadataPublisher: c.bucket, From 7056b2e0c4ca83afd7036daf8ad94a18ac70509e Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Wed, 28 Jul 2021 11:20:22 +0200 Subject: [PATCH 14/21] update hcp-sdk-go --- go.mod | 2 +- go.sum | 4 ++-- internal/packer_registry/registry.go | 15 +++++++-------- internal/packer_registry/types.bucket.go | 6 +++--- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 96d4b5bf1d9..05ad096260f 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/hashicorp/go-uuid v1.0.2 github.com/hashicorp/go-version v1.3.0 github.com/hashicorp/hcl/v2 v2.10.0 - github.com/hashicorp/hcp-sdk-go v0.10.1-0.20210717111725-9bb3c21afe93 + github.com/hashicorp/hcp-sdk-go v0.10.1-0.20210727200019-239ce8d80646 github.com/hashicorp/packer-plugin-alicloud v1.0.0 github.com/hashicorp/packer-plugin-amazon v1.0.0 github.com/hashicorp/packer-plugin-ansible v1.0.0 diff --git a/go.sum b/go.sum index 4c649e2304c..f6b66975cd7 100644 --- a/go.sum +++ b/go.sum @@ -675,8 +675,8 @@ github.com/hashicorp/hcl/v2 v2.8.0/go.mod h1:bQTN5mpo+jewjJgh8jr0JUguIi7qPHUF6yI github.com/hashicorp/hcl/v2 v2.9.1/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg= github.com/hashicorp/hcl/v2 v2.10.0 h1:1S1UnuhDGlv3gRFV4+0EdwB+znNP5HmcGbIqwnSCByg= github.com/hashicorp/hcl/v2 v2.10.0/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg= -github.com/hashicorp/hcp-sdk-go v0.10.1-0.20210717111725-9bb3c21afe93 h1:tCWmkmMKLZQIi53OpIAdqDw1PmtqbqFLBQYcY2sKIFQ= -github.com/hashicorp/hcp-sdk-go v0.10.1-0.20210717111725-9bb3c21afe93/go.mod h1:Tm9BAlTkp6jknZ0YNxF/556JBC/meCN1LUmWFN38HsU= +github.com/hashicorp/hcp-sdk-go v0.10.1-0.20210727200019-239ce8d80646 h1:4WbtYgGIxeRrGC0hGBtOi6XuJ23024H7r724rCL+eXI= +github.com/hashicorp/hcp-sdk-go v0.10.1-0.20210727200019-239ce8d80646/go.mod h1:Tm9BAlTkp6jknZ0YNxF/556JBC/meCN1LUmWFN38HsU= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= diff --git a/internal/packer_registry/registry.go b/internal/packer_registry/registry.go index 504b6e7a297..26f49b64b3d 100644 --- a/internal/packer_registry/registry.go +++ b/internal/packer_registry/registry.go @@ -4,7 +4,6 @@ import ( "context" "errors" - "github.com/go-openapi/runtime" packerSvc "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/preview/2021-04-30/client/packer_service" "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/preview/2021-04-30/models" "google.golang.org/grpc/codes" @@ -18,7 +17,7 @@ func CreateBucket(ctx context.Context, client *Client, input *models.HashicorpCl params.LocationProjectID = client.ProjectID params.Body = input - resp, err := client.Packer.CreateBucket(params, nil, func(*runtime.ClientOperation) {}) + resp, err := client.Packer.CreateBucket(params, nil) if err != nil { return "", err } @@ -48,7 +47,7 @@ func UpsertBucket(ctx context.Context, client *Client, input *models.HashicorpCl Description: input.Description, Labels: input.Labels, } - _, err = client.Packer.UpdateBucket(params, nil, func(*runtime.ClientOperation) {}) + _, err = client.Packer.UpdateBucket(params, nil) return err } @@ -67,7 +66,7 @@ func CreateIteration(ctx context.Context, client *Client, input *models.Hashicor params.BucketSlug = input.BucketSlug params.Body = input - it, err := client.Packer.CreateIteration(params, nil, func(*runtime.ClientOperation) {}) + it, err := client.Packer.CreateIteration(params, nil) if err != nil { return "", err } @@ -85,7 +84,7 @@ func GetIteration(ctx context.Context, client *Client, bucketslug string, finger // for now, we only care about fingerprint so we're hardcoding it. params.Fingerprint = &fingerprint - it, err := client.Packer.GetIteration(params, nil, func(*runtime.ClientOperation) {}) + it, err := client.Packer.GetIteration(params, nil) if err != nil { return "", err } @@ -101,7 +100,7 @@ func CreateBuild(ctx context.Context, client *Client, input *models.HashicorpClo params.BuildIterationID = input.Build.IterationID params.Body = input - resp, err := client.Packer.CreateBuild(params, nil, func(*runtime.ClientOperation) {}) + resp, err := client.Packer.CreateBuild(params, nil) if err != nil { return "", err } @@ -116,7 +115,7 @@ func ListBuilds(ctx context.Context, client *Client, bucketSlug string, iteratio params.BucketSlug = bucketSlug params.IterationID = iterationID - resp, err := client.Packer.ListBuilds(params, nil, func(*runtime.ClientOperation) {}) + resp, err := client.Packer.ListBuilds(params, nil) if err != nil { return []*models.HashicorpCloudPackerBuild{}, err } @@ -131,7 +130,7 @@ func UpdateBuild(ctx context.Context, client *Client, input *models.HashicorpClo params.LocationProjectID = client.ProjectID params.Body = input - resp, err := client.Packer.UpdateBuild(params, nil, func(*runtime.ClientOperation) {}) + resp, err := client.Packer.UpdateBuild(params, nil) if err != nil { return "", err } diff --git a/internal/packer_registry/types.bucket.go b/internal/packer_registry/types.bucket.go index aa520c14b57..afa1256922c 100644 --- a/internal/packer_registry/types.bucket.go +++ b/internal/packer_registry/types.bucket.go @@ -120,7 +120,7 @@ func (b *Bucket) Initialize(ctx context.Context) error { found = true log.Printf("build of component type %s already exists; skipping the create call", expected) - if *existing.Status == models.HashicorpCloudPackerBuildStatusDONE { + if existing.Status == models.HashicorpCloudPackerBuildStatusDONE { // We also need to remove the builds that are _complete_ from the // Iteration's expectedBuilds so we don't overwrite them. //b.Iteration.expectedBuilds = append(b.Iteration.expectedBuilds[:i], b.Iteration.expectedBuilds[i+1:]...) @@ -215,7 +215,7 @@ func (b *Bucket) PublishBuildStatus(ctx context.Context, name string, status mod Updates: &models.HashicorpCloudPackerBuildUpdates{ PackerRunUUID: buildToUpdate.RunUUID, Labels: buildToUpdate.Metadata, - Status: &status, + Status: status, }, } @@ -254,7 +254,7 @@ func (b *Bucket) CreateInitialBuildForIteration(ctx context.Context, name string ComponentType: name, IterationID: b.Iteration.ID, PackerRunUUID: b.Iteration.RunUUID, - Status: &status, + Status: status, }, } From 44a03934bf75cb6ebfeeb7d8542fe1c93af21b09 Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Wed, 28 Jul 2021 11:51:56 +0200 Subject: [PATCH 15/21] add hcp_packer_registry tests --- hcl2template/common_test.go | 8 +- .../complete.pkr.hcl} | 9 + .../testdata/hcp_par/duplicate.pkr.hcl | 11 + hcl2template/testdata/hcp_par/empty.pkr.hcl | 12 + hcl2template/testdata/hcp_par/invalid.pkr.hcl | 7 + .../testdata/hcp_par/long-description.pkr.hcl | 11 + hcl2template/types.build.go | 8 + .../types.build.hcp_packer_registry.go | 10 + .../types.build.hcp_packer_registry_test.go | 212 ++++++++++++++++++ hcl2template/types.build_test.go | 35 --- 10 files changed, 287 insertions(+), 36 deletions(-) rename hcl2template/testdata/{build/hcp_packer_registry.pkr.hcl => hcp_par/complete.pkr.hcl} (66%) create mode 100644 hcl2template/testdata/hcp_par/duplicate.pkr.hcl create mode 100644 hcl2template/testdata/hcp_par/empty.pkr.hcl create mode 100644 hcl2template/testdata/hcp_par/invalid.pkr.hcl create mode 100644 hcl2template/testdata/hcp_par/long-description.pkr.hcl create mode 100644 hcl2template/types.build.hcp_packer_registry_test.go diff --git a/hcl2template/common_test.go b/hcl2template/common_test.go index c9f7b978934..6159dbe76d6 100644 --- a/hcl2template/common_test.go +++ b/hcl2template/common_test.go @@ -6,7 +6,6 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "github.com/hashicorp/go-version" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclparse" @@ -14,6 +13,7 @@ import ( "github.com/hashicorp/packer-plugin-sdk/template/config" "github.com/hashicorp/packer/builder/null" . "github.com/hashicorp/packer/hcl2template/internal" + "github.com/hashicorp/packer/internal/packer_registry" "github.com/hashicorp/packer/packer" "github.com/zclconf/go-cty/cty" ) @@ -354,10 +354,16 @@ var cmpOpts = []cmp.Option{ packer.CoreBuildProvisioner{}, packer.CoreBuildPostProcessor{}, null.Builder{}, + packer_registry.Bucket{}, + packer_registry.Iteration{}, + packer.RegistryBuilder{}, ), cmpopts.IgnoreFields(PackerConfig{}, "Cwd", // Cwd will change for every os type ), + cmpopts.IgnoreFields(packer_registry.Iteration{}, + "Fingerprint", // Fingerprint will change everytime + ), cmpopts.IgnoreFields(VariableAssignment{}, "Expr", // its an interface ), diff --git a/hcl2template/testdata/build/hcp_packer_registry.pkr.hcl b/hcl2template/testdata/hcp_par/complete.pkr.hcl similarity index 66% rename from hcl2template/testdata/build/hcp_packer_registry.pkr.hcl rename to hcl2template/testdata/hcp_par/complete.pkr.hcl index 59339558606..ba4dd711125 100644 --- a/hcl2template/testdata/build/hcp_packer_registry.pkr.hcl +++ b/hcl2template/testdata/hcp_par/complete.pkr.hcl @@ -1,5 +1,7 @@ // starts resources to provision them. build { + name = "bucket-slug" + hcp_packer_registry { description = < 255 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf(buildHCPPackerRegistryLabel + ".description should have a maximum length of 255 characters"), + Subject: block.DefRange.Ptr(), + }) + return nil, diags + } + par.Description = b.Description par.Labels = b.Labels diff --git a/hcl2template/types.build.hcp_packer_registry_test.go b/hcl2template/types.build.hcp_packer_registry_test.go new file mode 100644 index 00000000000..47e6e4a1e72 --- /dev/null +++ b/hcl2template/types.build.hcp_packer_registry_test.go @@ -0,0 +1,212 @@ +package hcl2template + +import ( + "path/filepath" + "testing" + + packersdk "github.com/hashicorp/packer-plugin-sdk/packer" + "github.com/hashicorp/packer/internal/packer_registry" + "github.com/hashicorp/packer/packer" +) + +func Test_ParseHCPPackerRegistryBlock(t *testing.T) { + defaultParser := getBasicParser() + + tests := []parseTest{ + {"complete working build with hcp_packer_registry block", + defaultParser, + parseTestArgs{"testdata/hcp_par/complete.pkr.hcl", nil, nil}, + &PackerConfig{ + CorePackerVersionString: lockedVersion, + Basedir: filepath.Join("testdata", "hcp_par"), + Sources: map[SourceRef]SourceBlock{ + refVBIsoUbuntu1204: {Type: "virtualbox-iso", Name: "ubuntu-1204"}, + refAWSEBSUbuntu1604: {Type: "amazon-ebs", Name: "ubuntu-1604"}, + }, + Builds: Builds{ + &BuildBlock{ + Name: "bucket-slug", + HCPPackerRegistry: &HCPPackerRegistryBlock{ + Description: "Some description\n", + Labels: map[string]string{"foo": "bar"}, + }, + Sources: []SourceUseBlock{ + { + SourceRef: refVBIsoUbuntu1204, + }, + { + SourceRef: SourceRef{Type: "amazon-ebs", Name: "ubuntu-1604"}, + LocalName: "aws-ubuntu-16.04", + }, + }, + }, + }, + }, + false, false, + []packersdk.Build{ + &packer.CoreBuild{ + BuildName: "bucket-slug", + Type: "virtualbox-iso.ubuntu-1204", + Prepared: true, + Builder: &packer.RegistryBuilder{ + Name: "virtualbox-iso.ubuntu-1204", + Builder: emptyMockBuilder, + ArtifactMetadataPublisher: &packer_registry.Bucket{ + Slug: "bucket-slug", + Description: "Some description\n", + Labels: map[string]string{"foo": "bar"}, + Iteration: &packer_registry.Iteration{ + Fingerprint: "ignored-fingerprint", // this will be different everytime so it's ignored + }, + }, + }, + Provisioners: []packer.CoreBuildProvisioner{}, + PostProcessors: [][]packer.CoreBuildPostProcessor{ + { + { + PostProcessor: &packer.RegistryPostProcessor{ + BuilderType: "virtualbox-iso.ubuntu-1204", + ArtifactMetadataPublisher: &packer_registry.Bucket{ + Slug: "bucket-slug", + Description: "Some description\n", + Labels: map[string]string{"foo": "bar"}, + Iteration: &packer_registry.Iteration{ + Fingerprint: "ignored-fingerprint", + }, + }, + }, + }, + }, + }, + }, + &packer.CoreBuild{ + BuildName: "bucket-slug", + Type: "amazon-ebs.aws-ubuntu-16.04", + Prepared: true, + Builder: &packer.RegistryBuilder{ + Name: "amazon-ebs.aws-ubuntu-16.04", + Builder: emptyMockBuilder, + ArtifactMetadataPublisher: &packer_registry.Bucket{ + Slug: "bucket-slug", + Description: "Some description\n", + Labels: map[string]string{"foo": "bar"}, + Iteration: &packer_registry.Iteration{ + Fingerprint: "ignored-fingerprint", // this will be different everytime so it's ignored + }, + }, + }, + Provisioners: []packer.CoreBuildProvisioner{}, + PostProcessors: [][]packer.CoreBuildPostProcessor{ + { + { + PostProcessor: &packer.RegistryPostProcessor{ + BuilderType: "amazon-ebs.aws-ubuntu-16.04", + ArtifactMetadataPublisher: &packer_registry.Bucket{ + Slug: "bucket-slug", + Description: "Some description\n", + Labels: map[string]string{"foo": "bar"}, + Iteration: &packer_registry.Iteration{ + Fingerprint: "ignored-fingerprint", + }, + }, + }, + }, + }, + }, + }, + }, + false, + }, + {"duplicate hcp_packer_registry blocks", + defaultParser, + parseTestArgs{"testdata/hcp_par/duplicate.pkr.hcl", nil, nil}, + &PackerConfig{ + CorePackerVersionString: lockedVersion, + Basedir: filepath.Join("testdata", "hcp_par"), + }, + true, true, + nil, + false, + }, + {"empty hcp_packer_registry block", + defaultParser, + parseTestArgs{"testdata/hcp_par/empty.pkr.hcl", nil, nil}, + &PackerConfig{ + CorePackerVersionString: lockedVersion, + Basedir: filepath.Join("testdata", "hcp_par"), + Sources: map[SourceRef]SourceBlock{ + refVBIsoUbuntu1204: {Type: "virtualbox-iso", Name: "ubuntu-1204"}, + }, + Builds: Builds{ + &BuildBlock{ + Name: "bucket-slug", + HCPPackerRegistry: &HCPPackerRegistryBlock{}, + Sources: []SourceUseBlock{ + { + SourceRef: refVBIsoUbuntu1204, + }, + }, + }, + }, + }, + false, false, + []packersdk.Build{ + &packer.CoreBuild{ + BuildName: "bucket-slug", + Type: "virtualbox-iso.ubuntu-1204", + Prepared: true, + Builder: &packer.RegistryBuilder{ + Name: "virtualbox-iso.ubuntu-1204", + Builder: emptyMockBuilder, + ArtifactMetadataPublisher: &packer_registry.Bucket{ + Slug: "bucket-slug", + Iteration: &packer_registry.Iteration{ + Fingerprint: "ignored-fingerprint", // this will be different everytime so it's ignored + }, + }, + }, + Provisioners: []packer.CoreBuildProvisioner{}, + PostProcessors: [][]packer.CoreBuildPostProcessor{ + { + { + PostProcessor: &packer.RegistryPostProcessor{ + BuilderType: "virtualbox-iso.ubuntu-1204", + ArtifactMetadataPublisher: &packer_registry.Bucket{ + Slug: "bucket-slug", + Iteration: &packer_registry.Iteration{ + Fingerprint: "ignored-fingerprint", + }, + }, + }, + }, + }, + }, + }, + }, + false, + }, + {"invalid hcp_packer_registry config", + defaultParser, + parseTestArgs{"testdata/hcp_par/invalid.pkr.hcl", nil, nil}, + &PackerConfig{ + CorePackerVersionString: lockedVersion, + Basedir: filepath.Join("testdata", "hcp_par"), + }, + true, true, + nil, + false, + }, + {"long hcp_packer_registry.description", + defaultParser, + parseTestArgs{"testdata/hcp_par/long-description.pkr.hcl", nil, nil}, + &PackerConfig{ + CorePackerVersionString: lockedVersion, + Basedir: filepath.Join("testdata", "hcp_par"), + }, + true, true, + nil, + false, + }, + } + testParse(t, tests) +} diff --git a/hcl2template/types.build_test.go b/hcl2template/types.build_test.go index f70bee4c3d3..b2833375c39 100644 --- a/hcl2template/types.build_test.go +++ b/hcl2template/types.build_test.go @@ -421,41 +421,6 @@ func TestParse_build(t *testing.T) { }, false, }, - {"hcp packer registry build", - defaultParser, - parseTestArgs{"testdata/build/hcp_packer_registry.pkr.hcl", nil, nil}, - &PackerConfig{ - CorePackerVersionString: lockedVersion, - Basedir: filepath.Join("testdata", "build"), - Sources: map[SourceRef]SourceBlock{ - refVBIsoUbuntu1204: {Type: "virtualbox-iso", Name: "ubuntu-1204"}, - }, - Builds: Builds{ - &BuildBlock{ - HCPPackerRegistry: &HCPPackerRegistryBlock{ - Description: "Some description\n", - Labels: map[string]string{"foo": "bar"}, - }, - Sources: []SourceUseBlock{ - { - SourceRef: refVBIsoUbuntu1204, - }, - }, - }, - }, - }, - false, false, - []packersdk.Build{ - &packer.CoreBuild{ - Type: "virtualbox-iso.ubuntu-1204", - Prepared: true, - Builder: emptyMockBuilder, - Provisioners: []packer.CoreBuildProvisioner{}, - PostProcessors: [][]packer.CoreBuildPostProcessor{}, - }, - }, - false, - }, } testParse(t, tests) } From 83fbd8e498a93a40105fa07c744561442af4c9ac Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Wed, 28 Jul 2021 14:52:58 +0200 Subject: [PATCH 16/21] add some doc in the code --- hcl2template/testdata/hcp_par/duplicate.pkr.hcl | 2 +- hcl2template/testdata/hcp_par/empty.pkr.hcl | 1 + hcl2template/testdata/hcp_par/invalid.pkr.hcl | 2 +- hcl2template/testdata/hcp_par/long-description.pkr.hcl | 2 +- hcl2template/types.build.go | 1 + hcl2template/types.build.hcp_packer_registry.go | 7 +++++-- 6 files changed, 10 insertions(+), 5 deletions(-) diff --git a/hcl2template/testdata/hcp_par/duplicate.pkr.hcl b/hcl2template/testdata/hcp_par/duplicate.pkr.hcl index e3d850b05c1..1e6afe02bed 100644 --- a/hcl2template/testdata/hcp_par/duplicate.pkr.hcl +++ b/hcl2template/testdata/hcp_par/duplicate.pkr.hcl @@ -8,4 +8,4 @@ build { } } hcp_packer_registry {} -} \ No newline at end of file +} diff --git a/hcl2template/testdata/hcp_par/empty.pkr.hcl b/hcl2template/testdata/hcp_par/empty.pkr.hcl index 9af3e558bb1..da70ff87546 100644 --- a/hcl2template/testdata/hcp_par/empty.pkr.hcl +++ b/hcl2template/testdata/hcp_par/empty.pkr.hcl @@ -10,3 +10,4 @@ build { source "virtualbox-iso" "ubuntu-1204" { } + diff --git a/hcl2template/testdata/hcp_par/invalid.pkr.hcl b/hcl2template/testdata/hcp_par/invalid.pkr.hcl index 616e8c24914..42ba3210b94 100644 --- a/hcl2template/testdata/hcp_par/invalid.pkr.hcl +++ b/hcl2template/testdata/hcp_par/invalid.pkr.hcl @@ -4,4 +4,4 @@ build { hcp_packer_registry { labels = "" } -} \ No newline at end of file +} diff --git a/hcl2template/testdata/hcp_par/long-description.pkr.hcl b/hcl2template/testdata/hcp_par/long-description.pkr.hcl index 59fe33b3a88..3cc329b81b6 100644 --- a/hcl2template/testdata/hcp_par/long-description.pkr.hcl +++ b/hcl2template/testdata/hcp_par/long-description.pkr.hcl @@ -8,4 +8,4 @@ super super super super super super super super super super super super super su super super super long description EOT } -} \ No newline at end of file +} diff --git a/hcl2template/types.build.go b/hcl2template/types.build.go index 8f2b6320ae0..2ad40dc89e3 100644 --- a/hcl2template/types.build.go +++ b/hcl2template/types.build.go @@ -61,6 +61,7 @@ type BuildBlock struct { // call for example. Description string + // HCPPackerRegistry contains the configuration for publishing the image to the HCP Packer Registry. HCPPackerRegistry *HCPPackerRegistryBlock // Sources is the list of sources that we want to start in this build block. diff --git a/hcl2template/types.build.hcp_packer_registry.go b/hcl2template/types.build.hcp_packer_registry.go index 1e28ad7c1d3..819206e92fd 100644 --- a/hcl2template/types.build.hcp_packer_registry.go +++ b/hcl2template/types.build.hcp_packer_registry.go @@ -2,16 +2,19 @@ package hcl2template import ( "fmt" + "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" packerregistry "github.com/hashicorp/packer/internal/packer_registry" ) type HCPPackerRegistryBlock struct { + // Bucket description Description string - Labels map[string]string + // Bucket labels + Labels map[string]string - HCL2Ref HCL2Ref + HCL2Ref } func (b *HCPPackerRegistryBlock) WriteBucketConfig(bucket *packerregistry.Bucket) { From 81dad998c95e94571aaec1509734b0b69474720f Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Wed, 28 Jul 2021 15:09:54 +0200 Subject: [PATCH 17/21] add packer.io docs --- .../blocks/build/hcp_packer_registry.mdx | 47 +++++++++++++++++++ website/data/docs-nav-data.json | 4 ++ 2 files changed, 51 insertions(+) create mode 100644 website/content/docs/templates/hcl_templates/blocks/build/hcp_packer_registry.mdx diff --git a/website/content/docs/templates/hcl_templates/blocks/build/hcp_packer_registry.mdx b/website/content/docs/templates/hcl_templates/blocks/build/hcp_packer_registry.mdx new file mode 100644 index 00000000000..fff0c73aca6 --- /dev/null +++ b/website/content/docs/templates/hcl_templates/blocks/build/hcp_packer_registry.mdx @@ -0,0 +1,47 @@ +--- +description: > + The hcp_packer_registry allows operators the ability to customize the metadata sent to HCP Packer Registry. + It configures the base details of an image that is created or updated within HCP PAR. +page_title: hcp_packer_registry - build - Blocks +--- + +# The `hcp_packer_registry` block + +The `hcp_packer_registry` block allows operators the ability to customize the metadata sent to +HCP Packer Registry. It configures the details of an image that is created or updated within HCP PAR. + +The presence of a `hcp_packer_registry` block will enable HCP PAR mode and all the builds within that build block +will be pushed to the remote registry as far the appropriate HCP credentials are set (`HCP_CLIENT_ID` and `HCP_CLIENT_SECRET`). + +```hcl +# file: builds.pkr.hcl +source "file" "basic-example" { + content = "Lorem ipsum dolor sit amet" + target = "sample_artifact" +} + +build { + name = "example-artifact" + + hcp_packer_registry { + description = <hcp_packer_registry", + "path": "templates/hcl_templates/blocks/build/hcp_packer_registry" + }, { "title": "source", "path": "templates/hcl_templates/blocks/build/source" From 7a521fb2f660d67035b4785bfb9b311ab778b196 Mon Sep 17 00:00:00 2001 From: Sylvia Moss Date: Wed, 28 Jul 2021 16:49:09 +0200 Subject: [PATCH 18/21] Update website/content/docs/templates/hcl_templates/blocks/build/hcp_packer_registry.mdx Co-authored-by: Wilken Rivera --- .../hcl_templates/blocks/build/hcp_packer_registry.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/docs/templates/hcl_templates/blocks/build/hcp_packer_registry.mdx b/website/content/docs/templates/hcl_templates/blocks/build/hcp_packer_registry.mdx index fff0c73aca6..7db11a1d82d 100644 --- a/website/content/docs/templates/hcl_templates/blocks/build/hcp_packer_registry.mdx +++ b/website/content/docs/templates/hcl_templates/blocks/build/hcp_packer_registry.mdx @@ -11,7 +11,7 @@ The `hcp_packer_registry` block allows operators the ability to customize the me HCP Packer Registry. It configures the details of an image that is created or updated within HCP PAR. The presence of a `hcp_packer_registry` block will enable HCP PAR mode and all the builds within that build block -will be pushed to the remote registry as far the appropriate HCP credentials are set (`HCP_CLIENT_ID` and `HCP_CLIENT_SECRET`). +will be pushed to the remote registry if the appropriate HCP credentials are set (`HCP_CLIENT_ID` and `HCP_CLIENT_SECRET`). If no HCP credentials are set Packer will fail the build and exit immediately to avoid any potential artifact drift between the build providers and the Packer registry. ```hcl # file: builds.pkr.hcl From c8975e8259d65e7d818aa2104d8e325cab0f96db Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Wed, 28 Jul 2021 16:45:12 +0200 Subject: [PATCH 19/21] remove copy and paste --- hcl2template/testdata/hcp_par/complete.pkr.hcl | 1 - hcl2template/testdata/hcp_par/duplicate.pkr.hcl | 1 - hcl2template/testdata/hcp_par/empty.pkr.hcl | 1 - hcl2template/testdata/hcp_par/invalid.pkr.hcl | 1 - hcl2template/testdata/hcp_par/long-description.pkr.hcl | 1 - 5 files changed, 5 deletions(-) diff --git a/hcl2template/testdata/hcp_par/complete.pkr.hcl b/hcl2template/testdata/hcp_par/complete.pkr.hcl index ba4dd711125..4dda76fd3d6 100644 --- a/hcl2template/testdata/hcp_par/complete.pkr.hcl +++ b/hcl2template/testdata/hcp_par/complete.pkr.hcl @@ -1,4 +1,3 @@ -// starts resources to provision them. build { name = "bucket-slug" diff --git a/hcl2template/testdata/hcp_par/duplicate.pkr.hcl b/hcl2template/testdata/hcp_par/duplicate.pkr.hcl index 1e6afe02bed..1ed68757d8a 100644 --- a/hcl2template/testdata/hcp_par/duplicate.pkr.hcl +++ b/hcl2template/testdata/hcp_par/duplicate.pkr.hcl @@ -1,4 +1,3 @@ -// starts resources to provision them. build { name = "bucket-slug" hcp_packer_registry { diff --git a/hcl2template/testdata/hcp_par/empty.pkr.hcl b/hcl2template/testdata/hcp_par/empty.pkr.hcl index da70ff87546..b80f3c4541b 100644 --- a/hcl2template/testdata/hcp_par/empty.pkr.hcl +++ b/hcl2template/testdata/hcp_par/empty.pkr.hcl @@ -1,4 +1,3 @@ -// starts resources to provision them. build { name = "bucket-slug" hcp_packer_registry {} diff --git a/hcl2template/testdata/hcp_par/invalid.pkr.hcl b/hcl2template/testdata/hcp_par/invalid.pkr.hcl index 42ba3210b94..8634de19caf 100644 --- a/hcl2template/testdata/hcp_par/invalid.pkr.hcl +++ b/hcl2template/testdata/hcp_par/invalid.pkr.hcl @@ -1,4 +1,3 @@ -// starts resources to provision them. build { name = "bucket-slug" hcp_packer_registry { diff --git a/hcl2template/testdata/hcp_par/long-description.pkr.hcl b/hcl2template/testdata/hcp_par/long-description.pkr.hcl index 3cc329b81b6..c20502e1bd8 100644 --- a/hcl2template/testdata/hcp_par/long-description.pkr.hcl +++ b/hcl2template/testdata/hcp_par/long-description.pkr.hcl @@ -1,4 +1,3 @@ -// starts resources to provision them. build { name = "bucket-slug" hcp_packer_registry { From a953b653c8507e9cd90dece72d069f0dfc8b96a7 Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Wed, 28 Jul 2021 17:26:03 +0200 Subject: [PATCH 20/21] add slug option to hcp_packer_registry --- hcl2template/common_test.go | 8 +- .../hcp_par/build-description.pkr.hcl | 15 ++ .../override-build-description.pkr.hcl | 18 ++ hcl2template/testdata/hcp_par/slug.pkr.hcl | 20 ++ hcl2template/types.build.go | 20 +- .../types.build.hcp_packer_registry.go | 11 +- .../types.build.hcp_packer_registry_test.go | 186 ++++++++++++++++++ 7 files changed, 267 insertions(+), 11 deletions(-) create mode 100644 hcl2template/testdata/hcp_par/build-description.pkr.hcl create mode 100644 hcl2template/testdata/hcp_par/override-build-description.pkr.hcl create mode 100644 hcl2template/testdata/hcp_par/slug.pkr.hcl diff --git a/hcl2template/common_test.go b/hcl2template/common_test.go index 6159dbe76d6..1765a3fa4ab 100644 --- a/hcl2template/common_test.go +++ b/hcl2template/common_test.go @@ -13,7 +13,7 @@ import ( "github.com/hashicorp/packer-plugin-sdk/template/config" "github.com/hashicorp/packer/builder/null" . "github.com/hashicorp/packer/hcl2template/internal" - "github.com/hashicorp/packer/internal/packer_registry" + packerregistry "github.com/hashicorp/packer/internal/packer_registry" "github.com/hashicorp/packer/packer" "github.com/zclconf/go-cty/cty" ) @@ -354,14 +354,14 @@ var cmpOpts = []cmp.Option{ packer.CoreBuildProvisioner{}, packer.CoreBuildPostProcessor{}, null.Builder{}, - packer_registry.Bucket{}, - packer_registry.Iteration{}, + packerregistry.Bucket{}, + packerregistry.Iteration{}, packer.RegistryBuilder{}, ), cmpopts.IgnoreFields(PackerConfig{}, "Cwd", // Cwd will change for every os type ), - cmpopts.IgnoreFields(packer_registry.Iteration{}, + cmpopts.IgnoreFields(packerregistry.Iteration{}, "Fingerprint", // Fingerprint will change everytime ), cmpopts.IgnoreFields(VariableAssignment{}, diff --git a/hcl2template/testdata/hcp_par/build-description.pkr.hcl b/hcl2template/testdata/hcp_par/build-description.pkr.hcl new file mode 100644 index 00000000000..e4ec3410ce8 --- /dev/null +++ b/hcl2template/testdata/hcp_par/build-description.pkr.hcl @@ -0,0 +1,15 @@ +build { + description = < Date: Wed, 28 Jul 2021 17:31:23 +0200 Subject: [PATCH 21/21] add slug docs --- .../blocks/build/hcp_packer_registry.mdx | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/website/content/docs/templates/hcl_templates/blocks/build/hcp_packer_registry.mdx b/website/content/docs/templates/hcl_templates/blocks/build/hcp_packer_registry.mdx index 7db11a1d82d..8cd8bded3d8 100644 --- a/website/content/docs/templates/hcl_templates/blocks/build/hcp_packer_registry.mdx +++ b/website/content/docs/templates/hcl_templates/blocks/build/hcp_packer_registry.mdx @@ -11,7 +11,7 @@ The `hcp_packer_registry` block allows operators the ability to customize the me HCP Packer Registry. It configures the details of an image that is created or updated within HCP PAR. The presence of a `hcp_packer_registry` block will enable HCP PAR mode and all the builds within that build block -will be pushed to the remote registry if the appropriate HCP credentials are set (`HCP_CLIENT_ID` and `HCP_CLIENT_SECRET`). If no HCP credentials are set Packer will fail the build and exit immediately to avoid any potential artifact drift between the build providers and the Packer registry. +will be pushed to the remote registry if the appropriate HCP credentials are set (`HCP_CLIENT_ID` and `HCP_CLIENT_SECRET`). If no HCP credentials are set Packer will fail the build and exit immediately to avoid any potential artifact drift between the build providers and the Packer registry. ```hcl # file: builds.pkr.hcl @@ -21,9 +21,9 @@ source "file" "basic-example" { } build { - name = "example-artifact" - hcp_packer_registry { + slug = "sample-artifact" + description = <