From 12467b007732b83d0445024fd0fcb30cdfd44f18 Mon Sep 17 00:00:00 2001 From: Luis Davim Date: Mon, 8 Jul 2024 22:25:06 +0100 Subject: [PATCH] refactor: make importing templates available as a lib Signed-off-by: Luis Davim --- go.mod | 1 + go.sum | 2 + govc/flags/output.go | 118 +----- govc/importx/options.go | 74 +--- govc/importx/ova.go | 15 +- govc/importx/ovf.go | 337 ++---------------- govc/importx/spec.go | 139 +------- govc/library/import.go | 19 +- {govc/importx => ovf/importer}/archive.go | 53 +-- {govc/importx => ovf/importer}/importable.go | 6 +- ovf/importer/importer.go | 326 +++++++++++++++++ ovf/importer/options.go | 91 +++++ .../importx => ovf/importer}/options_test.go | 8 +- ovf/importer/spec.go | 138 +++++++ scripts/license.sh | 2 +- vim25/progress/loger.go | 125 +++++++ 16 files changed, 776 insertions(+), 678 deletions(-) rename {govc/importx => ovf/importer}/archive.go (69%) rename {govc/importx => ovf/importer}/importable.go (91%) create mode 100644 ovf/importer/importer.go create mode 100644 ovf/importer/options.go rename {govc/importx => ovf/importer}/options_test.go (90%) create mode 100644 ovf/importer/spec.go create mode 100644 vim25/progress/loger.go diff --git a/go.mod b/go.mod index 883b64d04..6cb237ed4 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728 github.com/xlab/treeprint v1.2.0 + golang.org/x/text v0.16.0 ) require ( diff --git a/go.sum b/go.sum index 17c83ddd2..b65864128 100644 --- a/go.sum +++ b/go.sum @@ -31,6 +31,8 @@ github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728 h1:sH9mEk+fly github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/govc/flags/output.go b/govc/flags/output.go index c0338b18c..8a2344b42 100644 --- a/govc/flags/output.go +++ b/govc/flags/output.go @@ -1,11 +1,11 @@ /* -Copyright (c) 2014-2016 VMware, Inc. All Rights Reserved. +Copyright (c) 2014-2024 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -26,7 +26,6 @@ import ( "os" "reflect" "strings" - "sync" "time" "github.com/dougm/pretty" @@ -204,7 +203,7 @@ type errorOutput struct { } func (e errorOutput) Write(w io.Writer) error { - reason := e.error.Error() + reason := e.Error() var messages []string var faults []types.LocalizableMessage @@ -261,15 +260,15 @@ func (e errorOutput) canEncode() bool { return soap.IsSoapFault(e.error) || soap.IsVimFault(e.error) } -// cannotEncode causes cli.Run to output err.Error() as it would without an error format specified -var cannotEncode = errors.New("cannot encode error") +// errCannotEncode causes cli.Run to output err.Error() as it would without an error format specified +var errCannotEncode = errors.New("cannot encode error") func (e errorOutput) MarshalJSON() ([]byte, error) { _, ok := e.error.(json.Marshaler) if ok || e.canEncode() { return json.Marshal(e.error) } - return nil, cannotEncode + return nil, errCannotEncode } func (e errorOutput) MarshalXML(encoder *xml.Encoder, start xml.StartElement) error { @@ -277,108 +276,9 @@ func (e errorOutput) MarshalXML(encoder *xml.Encoder, start xml.StartElement) er if ok || e.canEncode() { return encoder.Encode(e.error) } - return cannotEncode + return errCannotEncode } -type progressLogger struct { - flag *OutputFlag - prefix string - - wg sync.WaitGroup - - sink chan chan progress.Report - done chan struct{} -} - -func newProgressLogger(flag *OutputFlag, prefix string) *progressLogger { - p := &progressLogger{ - flag: flag, - prefix: prefix, - - sink: make(chan chan progress.Report), - done: make(chan struct{}), - } - - p.wg.Add(1) - - go p.loopA() - - return p -} - -// loopA runs before Sink() has been called. -func (p *progressLogger) loopA() { - var err error - - defer p.wg.Done() - - tick := time.NewTicker(100 * time.Millisecond) - defer tick.Stop() - - called := false - - for stop := false; !stop; { - select { - case ch := <-p.sink: - err = p.loopB(tick, ch) - stop = true - called = true - case <-p.done: - stop = true - case <-tick.C: - line := fmt.Sprintf("\r%s", p.prefix) - p.flag.Log(line) - } - } - - if err != nil && err != io.EOF { - p.flag.Log(fmt.Sprintf("\r%sError: %s\n", p.prefix, err)) - } else if called { - p.flag.Log(fmt.Sprintf("\r%sOK\n", p.prefix)) - } -} - -// loopA runs after Sink() has been called. -func (p *progressLogger) loopB(tick *time.Ticker, ch <-chan progress.Report) error { - var r progress.Report - var ok bool - var err error - - for ok = true; ok; { - select { - case r, ok = <-ch: - if !ok { - break - } - err = r.Error() - case <-tick.C: - line := fmt.Sprintf("\r%s", p.prefix) - if r != nil { - line += fmt.Sprintf("(%.0f%%", r.Percentage()) - detail := r.Detail() - if detail != "" { - line += fmt.Sprintf(", %s", detail) - } - line += ")" - } - p.flag.Log(line) - } - } - - return err -} - -func (p *progressLogger) Sink() chan<- progress.Report { - ch := make(chan progress.Report) - p.sink <- ch - return ch -} - -func (p *progressLogger) Wait() { - close(p.done) - p.wg.Wait() -} - -func (flag *OutputFlag) ProgressLogger(prefix string) *progressLogger { - return newProgressLogger(flag, prefix) +func (flag *OutputFlag) ProgressLogger(prefix string) *progress.ProgressLogger { + return progress.NewProgressLogger(flag.Log, prefix) } diff --git a/govc/importx/options.go b/govc/importx/options.go index 3d8190ec0..ddea6e388 100644 --- a/govc/importx/options.go +++ b/govc/importx/options.go @@ -1,11 +1,11 @@ /* -Copyright (c) 2015-2023 VMware, Inc. All Rights Reserved. +Copyright (c) 2015-2024 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -26,78 +26,12 @@ import ( "github.com/vmware/govmomi/govc/flags" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/ovf" + "github.com/vmware/govmomi/ovf/importer" "github.com/vmware/govmomi/vim25/types" ) -type KeyValue struct { - Key string - Value string -} - -// case insensitive for Key + Value -func (kv *KeyValue) UnmarshalJSON(b []byte) error { - e := struct { - types.KeyValue - Key *string - Value *string - }{ - types.KeyValue{}, &kv.Key, &kv.Value, - } - - err := json.Unmarshal(b, &e) - if err != nil { - return err - } - - if kv.Key == "" { - kv.Key = e.KeyValue.Key // "key" - } - - if kv.Value == "" { - kv.Value = e.KeyValue.Value // "value" - } - - return nil -} - -type Property struct { - KeyValue - Spec *ovf.Property `json:",omitempty"` -} - -type Network struct { - Name string - Network string -} - -type Options struct { - AllDeploymentOptions []string `json:",omitempty"` - Deployment string `json:",omitempty"` - - AllDiskProvisioningOptions []string `json:",omitempty"` - DiskProvisioning string - - AllIPAllocationPolicyOptions []string `json:",omitempty"` - IPAllocationPolicy string - - AllIPProtocolOptions []string `json:",omitempty"` - IPProtocol string - - PropertyMapping []Property `json:",omitempty"` - - NetworkMapping []Network `json:",omitempty"` - - Annotation string `json:",omitempty"` - - MarkAsTemplate bool - PowerOn bool - InjectOvfEnv bool - WaitForIP bool - Name *string -} - type OptionsFlag struct { - Options Options + Options importer.Options path string } diff --git a/govc/importx/ova.go b/govc/importx/ova.go index 343cdcdda..67712050f 100644 --- a/govc/importx/ova.go +++ b/govc/importx/ova.go @@ -1,11 +1,11 @@ /* -Copyright (c) 2014-2015 VMware, Inc. All Rights Reserved. +Copyright (c) 2014-2024 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -22,6 +22,7 @@ import ( "github.com/vmware/govmomi/govc/cli" "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/ovf/importer" "github.com/vmware/govmomi/vim25/types" ) @@ -43,21 +44,21 @@ func (cmd *ova) Run(ctx context.Context, f *flag.FlagSet) error { return err } - archive := &TapeArchive{Path: fpath} - archive.Client = cmd.Client + archive := &importer.TapeArchive{Path: fpath} + archive.Client = cmd.Importer.Client - cmd.Archive = archive + cmd.Importer.Archive = archive moref, err := cmd.Import(fpath) if err != nil { return err } - vm := object.NewVirtualMachine(cmd.Client, *moref) + vm := object.NewVirtualMachine(cmd.Importer.Client, *moref) return cmd.Deploy(vm, cmd.OutputFlag) } func (cmd *ova) Import(fpath string) (*types.ManagedObjectReference, error) { ovf := "*.ovf" - return cmd.ovfx.Import(ovf) + return cmd.Importer.Import(context.TODO(), ovf, cmd.Options) } diff --git a/govc/importx/ovf.go b/govc/importx/ovf.go index e1765f4cb..c903fc9a1 100644 --- a/govc/importx/ovf.go +++ b/govc/importx/ovf.go @@ -1,5 +1,5 @@ /* -Copyright (c) 2014-2023 VMware, Inc. All Rights Reserved. +Copyright (c) 2014-2024 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,23 +17,14 @@ limitations under the License. package importx import ( - "bytes" "context" "errors" "flag" - "fmt" - "path" - "strings" - "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/govc/cli" "github.com/vmware/govmomi/govc/flags" - "github.com/vmware/govmomi/nfc" "github.com/vmware/govmomi/object" - "github.com/vmware/govmomi/ovf" - "github.com/vmware/govmomi/vim25" - "github.com/vmware/govmomi/vim25/soap" - "github.com/vmware/govmomi/vim25/types" + "github.com/vmware/govmomi/ovf/importer" ) type ovfx struct { @@ -43,17 +34,9 @@ type ovfx struct { *flags.ResourcePoolFlag *flags.FolderFlag - *ArchiveFlag *OptionsFlag - Name string - VerifyManifest bool - Hidden bool - - Client *vim25.Client - Datacenter *object.Datacenter - Datastore *object.Datastore - ResourcePool *object.ResourcePool + Importer importer.Importer } func init() { @@ -72,14 +55,12 @@ func (cmd *ovfx) Register(ctx context.Context, f *flag.FlagSet) { cmd.FolderFlag, ctx = flags.NewFolderFlag(ctx) cmd.FolderFlag.Register(ctx, f) - cmd.ArchiveFlag, ctx = newArchiveFlag(ctx) - cmd.ArchiveFlag.Register(ctx, f) cmd.OptionsFlag, ctx = newOptionsFlag(ctx) cmd.OptionsFlag.Register(ctx, f) - f.StringVar(&cmd.Name, "name", "", "Name to use for new entity") - f.BoolVar(&cmd.VerifyManifest, "m", false, "Verify checksum of uploaded files against manifest (.mf)") - f.BoolVar(&cmd.Hidden, "hidden", false, "Enable hidden properties") + f.StringVar(&cmd.Importer.Name, "name", "", "Name to use for new entity") + f.BoolVar(&cmd.Importer.VerifyManifest, "m", false, "Verify checksum of uploaded files against manifest (.mf)") + f.BoolVar(&cmd.Importer.Hidden, "hidden", false, "Enable hidden properties") } func (cmd *ovfx) Process(ctx context.Context) error { @@ -95,9 +76,6 @@ func (cmd *ovfx) Process(ctx context.Context) error { if err := cmd.ResourcePoolFlag.Process(ctx); err != nil { return err } - if err := cmd.ArchiveFlag.Process(ctx); err != nil { - return err - } if err := cmd.OptionsFlag.Process(ctx); err != nil { return err } @@ -117,17 +95,17 @@ func (cmd *ovfx) Run(ctx context.Context, f *flag.FlagSet) error { return err } - archive := &FileArchive{Path: fpath} - archive.Client = cmd.Client + archive := &importer.FileArchive{Path: fpath} + archive.Client = cmd.Importer.Client - cmd.Archive = archive + cmd.Importer.Archive = archive - moref, err := cmd.Import(fpath) + moref, err := cmd.Importer.Import(context.TODO(), fpath, cmd.Options) if err != nil { return err } - vm := object.NewVirtualMachine(cmd.Client, *moref) + vm := object.NewVirtualMachine(cmd.Importer.Client, *moref) return cmd.Deploy(vm, cmd.OutputFlag) } @@ -139,313 +117,70 @@ func (cmd *ovfx) Prepare(f *flag.FlagSet) (string, error) { return "", errors.New("no file specified") } - cmd.Client, err = cmd.DatastoreFlag.Client() + cmd.Importer.Log = cmd.OutputFlag.Log + cmd.Importer.Client, err = cmd.DatastoreFlag.Client() if err != nil { return "", err } - cmd.Datacenter, err = cmd.DatastoreFlag.Datacenter() + cmd.Importer.Datacenter, err = cmd.DatastoreFlag.Datacenter() if err != nil { return "", err } - cmd.Datastore, err = cmd.DatastoreFlag.Datastore() + cmd.Importer.Datastore, err = cmd.DatastoreFlag.Datastore() if err != nil { return "", err } - cmd.ResourcePool, err = cmd.ResourcePoolFlag.ResourcePoolIfSpecified() + cmd.Importer.ResourcePool, err = cmd.ResourcePoolIfSpecified() if err != nil { return "", err } - return f.Arg(0), nil -} - -func (cmd *ovfx) Map(op []Property) (p []types.KeyValue) { - for _, v := range op { - p = append(p, types.KeyValue{ - Key: v.Key, - Value: v.Value, - }) - } - - return -} - -func (cmd *ovfx) validateNetwork(e *ovf.Envelope, net Network) { - var names []string - - if e.Network != nil { - for _, n := range e.Network.Networks { - if n.Name == net.Name { - return - } - names = append(names, n.Name) - } - } - - _, _ = cmd.Log(fmt.Sprintf("Warning: invalid NetworkMapping.Name=%q, valid names=%s\n", net.Name, names)) -} - -func (cmd *ovfx) NetworkMap(e *ovf.Envelope) ([]types.OvfNetworkMapping, error) { - ctx := context.TODO() - finder, err := cmd.DatastoreFlag.Finder() - if err != nil { - return nil, err - } - - var nmap []types.OvfNetworkMapping - for _, m := range cmd.Options.NetworkMapping { - if m.Network == "" { - continue // Not set, let vSphere choose the default network - } - cmd.validateNetwork(e, m) - - var ref types.ManagedObjectReference - - net, err := finder.Network(ctx, m.Network) - if err != nil { - switch err.(type) { - case *find.NotFoundError: - if !ref.FromString(m.Network) { - return nil, err - } // else this is a raw MO ref - default: - return nil, err - } - } else { - ref = net.Reference() - } - - nmap = append(nmap, types.OvfNetworkMapping{ - Name: m.Name, - Network: ref, - }) - } - - return nmap, err -} - -func (cmd *ovfx) Import(fpath string) (*types.ManagedObjectReference, error) { - ctx := context.TODO() - - o, err := cmd.ReadOvf(fpath) - if err != nil { - return nil, err - } - - e, err := cmd.ReadEnvelope(o) - if err != nil { - return nil, fmt.Errorf("failed to parse ovf: %s", err) - } - - name := "Govc Virtual Appliance" - if e.VirtualSystem != nil { - name = e.VirtualSystem.ID - if e.VirtualSystem.Name != nil { - name = *e.VirtualSystem.Name - } - - if cmd.Hidden { - // TODO: userConfigurable is optional and defaults to false, so we should *add* userConfigurable=true - // if not set for a Property. But, there'd be a bunch more work involved to preserve other data in doing - // a complete xml.Marshal of the .ovf - o = bytes.ReplaceAll(o, []byte(`userConfigurable="false"`), []byte(`userConfigurable="true"`)) - } - } - - // Override name from options if specified - if cmd.Options.Name != nil { - name = *cmd.Options.Name - } - - // Override name from arguments if specified - if cmd.Name != "" { - name = cmd.Name - } - - nmap, err := cmd.NetworkMap(e) - if err != nil { - return nil, err - } - - cisp := types.OvfCreateImportSpecParams{ - DiskProvisioning: cmd.Options.DiskProvisioning, - EntityName: name, - IpAllocationPolicy: cmd.Options.IPAllocationPolicy, - IpProtocol: cmd.Options.IPProtocol, - OvfManagerCommonParams: types.OvfManagerCommonParams{ - DeploymentOption: cmd.Options.Deployment, - Locale: "US"}, - PropertyMapping: cmd.Map(cmd.Options.PropertyMapping), - NetworkMapping: nmap, - } - host, err := cmd.HostSystemIfSpecified() if err != nil { - return nil, err + return "", err } - if cmd.ResourcePool == nil { + if cmd.Importer.ResourcePool == nil { if host == nil { - cmd.ResourcePool, err = cmd.ResourcePoolFlag.ResourcePool() + cmd.Importer.ResourcePool, err = cmd.ResourcePoolFlag.ResourcePool() } else { - cmd.ResourcePool, err = host.ResourcePool(ctx) + cmd.Importer.ResourcePool, err = host.ResourcePool(context.TODO()) } if err != nil { - return nil, err + return "", err } } - m := ovf.NewManager(cmd.Client) - spec, err := m.CreateImportSpec(ctx, string(o), cmd.ResourcePool, cmd.Datastore, cisp) + cmd.Importer.Finder, err = cmd.DatastoreFlag.Finder() if err != nil { - return nil, err - } - if spec.Error != nil { - return nil, errors.New(spec.Error[0].LocalizedMessage) - } - if spec.Warning != nil { - for _, w := range spec.Warning { - _, _ = cmd.Log(fmt.Sprintf("Warning: %s\n", w.LocalizedMessage)) - } + return "", err } - if cmd.Options.Annotation != "" { - switch s := spec.ImportSpec.(type) { - case *types.VirtualMachineImportSpec: - s.ConfigSpec.Annotation = cmd.Options.Annotation - case *types.VirtualAppImportSpec: - s.VAppConfigSpec.Annotation = cmd.Options.Annotation - } + cmd.Importer.Host, err = cmd.HostSystemIfSpecified() + if err != nil { + return "", err } - var folder *object.Folder // The folder argument must not be set on a VM in a vApp, otherwise causes // InvalidArgument fault: A specified parameter was not correct: pool - if cmd.ResourcePool.Reference().Type != "VirtualApp" { - folder, err = cmd.FolderOrDefault("vm") + if cmd.Importer.ResourcePool.Reference().Type != "VirtualApp" { + cmd.Importer.Folder, err = cmd.FolderOrDefault("vm") if err != nil { - return nil, err + return "", err } } - if cmd.VerifyManifest { - err = cmd.readManifest(fpath) - if err != nil { - return nil, err + if cmd.Importer.Name == "" { + // Override name from options if specified + if cmd.Options.Name != nil { + cmd.Importer.Name = *cmd.Options.Name } + } else { + cmd.Options.Name = &cmd.Importer.Name } - lease, err := cmd.ResourcePool.ImportVApp(ctx, spec.ImportSpec, folder, host) - if err != nil { - return nil, err - } - - info, err := lease.Wait(ctx, spec.FileItem) - if err != nil { - return nil, err - } - - u := lease.StartUpdater(ctx, info) - defer u.Done() - - for _, i := range info.Items { - err = cmd.Upload(ctx, lease, i) - if err != nil { - return nil, err - } - } - - return &info.Entity, lease.Complete(ctx) -} - -func (cmd *ovfx) Upload(ctx context.Context, lease *nfc.Lease, item nfc.FileItem) error { - file := item.Path - - f, size, err := cmd.Open(file) - if err != nil { - return err - } - defer f.Close() - - logger := cmd.ProgressLogger(fmt.Sprintf("Uploading %s... ", path.Base(file))) - defer logger.Wait() - - opts := soap.Upload{ - ContentLength: size, - Progress: logger, - } - - err = lease.Upload(ctx, item, f, opts) - if err != nil { - return err - } - - if cmd.VerifyManifest { - mapImportKeyToKey := func(urls []types.HttpNfcLeaseDeviceUrl, importKey string) string { - for _, url := range urls { - if url.ImportKey == importKey { - return url.Key - } - } - return "" - } - leaseInfo, err := lease.Wait(ctx, nil) - if err != nil { - return err - } - return cmd.validateChecksum(ctx, lease, file, mapImportKeyToKey(leaseInfo.DeviceUrl, item.DeviceId)) - } - return nil -} - -func (cmd *ovfx) validateChecksum(ctx context.Context, lease *nfc.Lease, file string, key string) error { - sum, found := cmd.manifest[file] - if !found { - msg := fmt.Sprintf("missing checksum for %v in manifest file", file) - return errors.New(msg) - } - // Perform the checksum match eagerly, after each file upload, instead - // of after uploading all the files, to provide fail-fast behavior. - // (Trade-off here is multiple GetManifest() API calls to the server.) - manifests, err := lease.GetManifest(ctx) - if err != nil { - return err - } - for _, m := range manifests { - if m.Key == key { - // Compare server-side computed checksum of uploaded file - // against the client's manifest entry (assuming client's - // manifest has correct checksums - client doesn't compute - // checksum of the file before uploading). - - // Try matching sha1 first (newer versions have moved to sha256). - if strings.ToUpper(sum.Algorithm) == "SHA1" { - if sum.Checksum != m.Sha1 { - msg := fmt.Sprintf("manifest checksum %v mismatch with uploaded checksum %v for file %v", - sum.Checksum, m.Sha1, file) - return errors.New(msg) - } - // Uploaded file checksum computed by server matches with local manifest entry. - return nil - } - // If not sha1, check for other types (in a separate field). - if !strings.EqualFold(sum.Algorithm, m.ChecksumType) { - msg := fmt.Sprintf("manifest checksum type %v mismatch with uploaded checksum type %v for file %v", - sum.Algorithm, m.ChecksumType, file) - return errors.New(msg) - } - if !strings.EqualFold(sum.Checksum, m.Checksum) { - msg := fmt.Sprintf("manifest checksum %v mismatch with uploaded checksum %v for file %v", - sum.Checksum, m.Checksum, file) - return errors.New(msg) - } - // Uploaded file checksum computed by server matches with local manifest entry. - return nil - } - } - msg := fmt.Sprintf("missing manifest entry on server for uploaded file %v (key %v), manifests=%#v", file, key, manifests) - return errors.New(msg) + return f.Arg(0), nil } diff --git a/govc/importx/spec.go b/govc/importx/spec.go index acb1729f1..01dcf0b69 100644 --- a/govc/importx/spec.go +++ b/govc/importx/spec.go @@ -22,27 +22,18 @@ import ( "fmt" "io" "path" - "strings" "github.com/vmware/govmomi/govc/cli" "github.com/vmware/govmomi/govc/flags" - "github.com/vmware/govmomi/ovf" - "github.com/vmware/govmomi/vim25/types" -) - -var ( - allDiskProvisioningOptions = types.OvfCreateImportSpecParamsDiskProvisioningType("").Strings() - - allIPAllocationPolicyOptions = types.VAppIPAssignmentInfoIpAllocationPolicy("").Strings() - - allIPProtocolOptions = types.VAppIPAssignmentInfoProtocols("").Strings() + "github.com/vmware/govmomi/ovf/importer" ) type spec struct { - *ArchiveFlag *flags.ClientFlag *flags.OutputFlag + Archive importer.Archive + hidden bool } @@ -51,8 +42,6 @@ func init() { } func (cmd *spec) Register(ctx context.Context, f *flag.FlagSet) { - cmd.ArchiveFlag, ctx = newArchiveFlag(ctx) - cmd.ArchiveFlag.Register(ctx, f) cmd.ClientFlag, ctx = flags.NewClientFlag(ctx) cmd.ClientFlag.Register(ctx, f) @@ -63,9 +52,6 @@ func (cmd *spec) Register(ctx context.Context, f *flag.FlagSet) { } func (cmd *spec) Process(ctx context.Context) error { - if err := cmd.ArchiveFlag.Process(ctx); err != nil { - return err - } if err := cmd.ClientFlag.Process(ctx); err != nil { return err } @@ -86,29 +72,29 @@ func (cmd *spec) Run(ctx context.Context, f *flag.FlagSet) error { if len(fpath) > 0 { switch path.Ext(fpath) { case ".ovf": - cmd.Archive = &FileArchive{Path: fpath} + cmd.Archive = &importer.FileArchive{Path: fpath} case "", ".ova": - cmd.Archive = &TapeArchive{Path: fpath} + cmd.Archive = &importer.TapeArchive{Path: fpath} fpath = "*.ovf" default: return fmt.Errorf("invalid file extension %s", path.Ext(fpath)) } - if isRemotePath(f.Arg(0)) { + if importer.IsRemotePath(f.Arg(0)) { client, err := cmd.Client() if err != nil { return err } switch archive := cmd.Archive.(type) { - case *FileArchive: + case *importer.FileArchive: archive.Client = client - case *TapeArchive: + case *importer.TapeArchive: archive.Client = client } } } - env, err := cmd.Spec(fpath) + env, err := importer.Spec(fpath, cmd.Archive, cmd.hidden, cmd.Verbose()) if err != nil { return err } @@ -120,114 +106,9 @@ func (cmd *spec) Run(ctx context.Context, f *flag.FlagSet) error { } type specResult struct { - *Options + *importer.Options } func (*specResult) Write(w io.Writer) error { return nil } - -func (cmd *spec) Map(e *ovf.Envelope) (res []Property) { - if e == nil || e.VirtualSystem == nil { - return nil - } - - for _, p := range e.VirtualSystem.Product { - for i, v := range p.Property { - if v.UserConfigurable == nil { - continue - } - if !*v.UserConfigurable && !cmd.hidden { - continue - } - - d := "" - if v.Default != nil { - d = *v.Default - } - - // vSphere only accept True/False as boolean values for some reason - if v.Type == "boolean" { - d = strings.Title(d) - } - - np := Property{KeyValue: KeyValue{Key: p.Key(v), Value: d}} - - if cmd.Verbose() { - np.Spec = &p.Property[i] - } - - res = append(res, np) - } - } - - return -} - -func (cmd *spec) Spec(fpath string) (*Options, error) { - e := &ovf.Envelope{} - if fpath != "" { - d, err := cmd.ReadOvf(fpath) - if err != nil { - return nil, err - } - - if e, err = cmd.ReadEnvelope(d); err != nil { - return nil, err - } - } - - var deploymentOptions []string - if e.DeploymentOption != nil && e.DeploymentOption.Configuration != nil { - // add default first - for _, c := range e.DeploymentOption.Configuration { - if c.Default != nil && *c.Default { - deploymentOptions = append(deploymentOptions, c.ID) - } - } - - for _, c := range e.DeploymentOption.Configuration { - if c.Default == nil || !*c.Default { - deploymentOptions = append(deploymentOptions, c.ID) - } - } - } - - o := Options{ - DiskProvisioning: allDiskProvisioningOptions[0], - IPAllocationPolicy: allIPAllocationPolicyOptions[0], - IPProtocol: allIPProtocolOptions[0], - MarkAsTemplate: false, - PowerOn: false, - WaitForIP: false, - InjectOvfEnv: false, - PropertyMapping: cmd.Map(e), - } - - if deploymentOptions != nil { - o.Deployment = deploymentOptions[0] - } - - if e.VirtualSystem != nil && e.VirtualSystem.Annotation != nil { - for _, a := range e.VirtualSystem.Annotation { - o.Annotation += a.Annotation - } - } - - if e.Network != nil { - for _, net := range e.Network.Networks { - o.NetworkMapping = append(o.NetworkMapping, Network{net.Name, ""}) - } - } - - if cmd.Verbose() { - if deploymentOptions != nil { - o.AllDeploymentOptions = deploymentOptions - } - o.AllDiskProvisioningOptions = allDiskProvisioningOptions - o.AllIPAllocationPolicyOptions = allIPAllocationPolicyOptions - o.AllIPProtocolOptions = allIPProtocolOptions - } - - return &o, nil -} diff --git a/govc/library/import.go b/govc/library/import.go index 4b18d84f9..a5f54e684 100644 --- a/govc/library/import.go +++ b/govc/library/import.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -29,7 +29,7 @@ import ( "github.com/vmware/govmomi/govc/cli" "github.com/vmware/govmomi/govc/flags" - "github.com/vmware/govmomi/govc/importx" + "github.com/vmware/govmomi/ovf/importer" "github.com/vmware/govmomi/vapi/library" "github.com/vmware/govmomi/vim25/soap" ) @@ -112,8 +112,9 @@ func (cmd *item) Run(ctx context.Context, f *flag.FlagSet) error { if err != nil { return err } - opener := importx.Opener{Client: client} - archive := &importx.ArchiveFlag{Archive: &importx.FileArchive{Path: file, Opener: opener}} + opener := importer.Opener{Client: client} + var archive importer.Archive + archive = &importer.FileArchive{Path: file, Opener: opener} manifest := make(map[string]*library.Checksum) if cmd.Name == "" { @@ -122,7 +123,7 @@ func (cmd *item) Run(ctx context.Context, f *flag.FlagSet) error { switch ext { case ".ova": - archive.Archive = &importx.TapeArchive{Path: file, Opener: opener} + archive = &importer.TapeArchive{Path: file, Opener: opener} base = "*.ovf" mf = "*.mf" kind = library.ItemTypeOVF @@ -206,7 +207,7 @@ func (cmd *item) Run(ctx context.Context, f *flag.FlagSet) error { } defer f.Close() - if e, ok := f.(*importx.TapeArchiveEntry); ok { + if e, ok := f.(*importer.TapeArchiveEntry); ok { name = e.Name // expand path.Match's (e.g. "*.ovf" -> "name.ovf") } @@ -228,7 +229,7 @@ func (cmd *item) Run(ctx context.Context, f *flag.FlagSet) error { if err != nil { return err } - if cmd.OutputFlag.TTY { + if cmd.TTY { logger := cmd.ProgressLogger(fmt.Sprintf("Uploading %s... ", name)) p.Progress = logger defer logger.Wait() @@ -241,12 +242,12 @@ func (cmd *item) Run(ctx context.Context, f *flag.FlagSet) error { } if cmd.Type == library.ItemTypeOVF { - o, err := archive.ReadOvf(base) + o, err := importer.ReadOvf(base, archive) if err != nil { return err } - e, err := archive.ReadEnvelope(o) + e, err := importer.ReadEnvelope(o) if err != nil { return fmt.Errorf("failed to parse ovf: %s", err) } diff --git a/govc/importx/archive.go b/ovf/importer/archive.go similarity index 69% rename from govc/importx/archive.go rename to ovf/importer/archive.go index fef36abdc..6b3f631ff 100644 --- a/govc/importx/archive.go +++ b/ovf/importer/archive.go @@ -1,11 +1,11 @@ /* -Copyright (c) 2014-2015 VMware, Inc. All Rights Reserved. +Copyright (c) 2024-2024 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -14,14 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package importx +package importer import ( "archive/tar" "bytes" "context" "errors" - "flag" "fmt" "io" "net/url" @@ -31,32 +30,12 @@ import ( "strings" "github.com/vmware/govmomi/ovf" - "github.com/vmware/govmomi/vapi/library" "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/soap" ) -// ArchiveFlag doesn't register any flags; -// only encapsulates some common archive related functionality. -type ArchiveFlag struct { - Archive - - manifest map[string]*library.Checksum -} - -func newArchiveFlag(ctx context.Context) (*ArchiveFlag, context.Context) { - return &ArchiveFlag{}, ctx -} - -func (f *ArchiveFlag) Register(ctx context.Context, fs *flag.FlagSet) { -} - -func (f *ArchiveFlag) Process(ctx context.Context) error { - return nil -} - -func (f *ArchiveFlag) ReadOvf(fpath string) ([]byte, error) { - r, _, err := f.Open(fpath) +func ReadOvf(fpath string, a Archive) ([]byte, error) { + r, _, err := a.Open(fpath) if err != nil { return nil, err } @@ -65,7 +44,7 @@ func (f *ArchiveFlag) ReadOvf(fpath string) ([]byte, error) { return io.ReadAll(r) } -func (f *ArchiveFlag) ReadEnvelope(data []byte) (*ovf.Envelope, error) { +func ReadEnvelope(data []byte) (*ovf.Envelope, error) { e, err := ovf.Unmarshal(bytes.NewReader(data)) if err != nil { return nil, fmt.Errorf("failed to parse ovf: %s", err) @@ -74,22 +53,6 @@ func (f *ArchiveFlag) ReadEnvelope(data []byte) (*ovf.Envelope, error) { return e, nil } -func (f *ArchiveFlag) readManifest(fpath string) error { - base := filepath.Base(fpath) - ext := filepath.Ext(base) - mfName := strings.Replace(base, ext, ".mf", 1) - - mf, _, err := f.Open(mfName) - if err != nil { - msg := fmt.Sprintf("manifest %q: %s", mf, err) - fmt.Fprintln(os.Stderr, msg) - return errors.New(msg) - } - f.manifest, err = library.ReadManifest(mf) - _ = mf.Close() - return err -} - type Archive interface { Open(string) (io.ReadCloser, int64, error) } @@ -163,7 +126,7 @@ type Opener struct { *vim25.Client } -func isRemotePath(path string) bool { +func IsRemotePath(path string) bool { if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { return true } @@ -185,7 +148,7 @@ func (o Opener) OpenLocal(path string) (io.ReadCloser, int64, error) { } func (o Opener) OpenFile(path string) (io.ReadCloser, int64, error) { - if isRemotePath(path) { + if IsRemotePath(path) { return o.OpenRemote(path) } return o.OpenLocal(path) diff --git a/govc/importx/importable.go b/ovf/importer/importable.go similarity index 91% rename from govc/importx/importable.go rename to ovf/importer/importable.go index 14e31670b..a43c7fc52 100644 --- a/govc/importx/importable.go +++ b/ovf/importer/importable.go @@ -1,11 +1,11 @@ /* -Copyright (c) 2014 VMware, Inc. All Rights Reserved. +Copyright (c) 2024-2024 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package importx +package importer import ( "fmt" diff --git a/ovf/importer/importer.go b/ovf/importer/importer.go new file mode 100644 index 000000000..55382f06c --- /dev/null +++ b/ovf/importer/importer.go @@ -0,0 +1,326 @@ +/* +Copyright (c) 2024-2024 VMware, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package importer + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/nfc" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/ovf" + "github.com/vmware/govmomi/vapi/library" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/progress" + "github.com/vmware/govmomi/vim25/soap" + "github.com/vmware/govmomi/vim25/types" +) + +type Importer struct { + Log progress.LogFunc + + Name string + VerifyManifest bool + Hidden bool + + Client *vim25.Client + Finder *find.Finder + Sinker progress.Sinker + + Datacenter *object.Datacenter + Datastore *object.Datastore + ResourcePool *object.ResourcePool + Host *object.HostSystem + Folder *object.Folder + + Archive Archive + Manifest map[string]*library.Checksum +} + +func (imp *Importer) ReadManifest(fpath string) error { + base := filepath.Base(fpath) + ext := filepath.Ext(base) + mfName := strings.Replace(base, ext, ".mf", 1) + + mf, _, err := imp.Archive.Open(mfName) + if err != nil { + msg := fmt.Sprintf("manifest %q: %s", mf, err) + fmt.Fprintln(os.Stderr, msg) + return errors.New(msg) + } + imp.Manifest, err = library.ReadManifest(mf) + _ = mf.Close() + return err +} + +func (imp *Importer) Import(ctx context.Context, fpath string, opts Options) (*types.ManagedObjectReference, error) { + + o, err := ReadOvf(fpath, imp.Archive) + if err != nil { + return nil, err + } + + e, err := ReadEnvelope(o) + if err != nil { + return nil, fmt.Errorf("failed to parse ovf: %s", err) + } + + if e.VirtualSystem != nil { + if e.VirtualSystem != nil { + if opts.Name == nil { + opts.Name = &e.VirtualSystem.ID + if e.VirtualSystem.Name != nil { + opts.Name = e.VirtualSystem.Name + } + } + } + if imp.Hidden { + // TODO: userConfigurable is optional and defaults to false, so we should *add* userConfigurable=true + // if not set for a Property. But, there'd be a bunch more work involved to preserve other data in doing + // a complete xml.Marshal of the .ovf + o = bytes.ReplaceAll(o, []byte(`userConfigurable="false"`), []byte(`userConfigurable="true"`)) + } + } + + name := "Govc Virtual Appliance" + if opts.Name != nil { + name = *opts.Name + } + + nmap, err := imp.NetworkMap(ctx, e, opts.NetworkMapping) + if err != nil { + return nil, err + } + + cisp := types.OvfCreateImportSpecParams{ + DiskProvisioning: opts.DiskProvisioning, + EntityName: name, + IpAllocationPolicy: opts.IPAllocationPolicy, + IpProtocol: opts.IPProtocol, + OvfManagerCommonParams: types.OvfManagerCommonParams{ + DeploymentOption: opts.Deployment, + Locale: "US"}, + PropertyMapping: OVFMap(opts.PropertyMapping), + NetworkMapping: nmap, + } + + m := ovf.NewManager(imp.Client) + spec, err := m.CreateImportSpec(ctx, string(o), imp.ResourcePool, imp.Datastore, cisp) + if err != nil { + return nil, err + } + if spec.Error != nil { + return nil, errors.New(spec.Error[0].LocalizedMessage) + } + if spec.Warning != nil { + for _, w := range spec.Warning { + _, _ = imp.Log(fmt.Sprintf("Warning: %s\n", w.LocalizedMessage)) + } + } + + if opts.Annotation != "" { + switch s := spec.ImportSpec.(type) { + case *types.VirtualMachineImportSpec: + s.ConfigSpec.Annotation = opts.Annotation + case *types.VirtualAppImportSpec: + s.VAppConfigSpec.Annotation = opts.Annotation + } + } + + if imp.VerifyManifest { + if err := imp.ReadManifest(fpath); err != nil { + return nil, err + } + } + + lease, err := imp.ResourcePool.ImportVApp(ctx, spec.ImportSpec, imp.Folder, imp.Host) + if err != nil { + return nil, err + } + + info, err := lease.Wait(ctx, spec.FileItem) + if err != nil { + return nil, err + } + + u := lease.StartUpdater(ctx, info) + defer u.Done() + + for _, i := range info.Items { + if err := imp.Upload(ctx, lease, i); err != nil { + return nil, err + } + } + + return &info.Entity, lease.Complete(ctx) +} + +func (imp *Importer) NetworkMap(ctx context.Context, e *ovf.Envelope, networks []Network) ([]types.OvfNetworkMapping, error) { + var nmap []types.OvfNetworkMapping + for _, m := range networks { + if m.Network == "" { + continue // Not set, let vSphere choose the default network + } + if err := ValidateNetwork(e, m); err != nil && imp.Log != nil { + _, _ = imp.Log(err.Error() + "\n") + } + + var ref types.ManagedObjectReference + + net, err := imp.Finder.Network(ctx, m.Network) + if err != nil { + switch err.(type) { + case *find.NotFoundError: + if !ref.FromString(m.Network) { + return nil, err + } // else this is a raw MO ref + default: + return nil, err + } + } else { + ref = net.Reference() + } + + nmap = append(nmap, types.OvfNetworkMapping{ + Name: m.Name, + Network: ref, + }) + } + + return nmap, nil +} + +func OVFMap(op []Property) (p []types.KeyValue) { + for _, v := range op { + p = append(p, types.KeyValue{ + Key: v.Key, + Value: v.Value, + }) + } + + return +} + +func ValidateNetwork(e *ovf.Envelope, net Network) error { + var names []string + + if e.Network != nil { + for _, n := range e.Network.Networks { + if n.Name == net.Name { + return nil + } + names = append(names, n.Name) + } + } + + return fmt.Errorf("warning: invalid NetworkMapping.Name=%q, valid names=%s", net.Name, names) +} + +func ValidateChecksum(ctx context.Context, lease *nfc.Lease, sum *library.Checksum, file string, key string) error { + // Perform the checksum match eagerly, after each file upload, instead + // of after uploading all the files, to provide fail-fast behavior. + // (Trade-off here is multiple GetManifest() API calls to the server.) + manifests, err := lease.GetManifest(ctx) + if err != nil { + return err + } + for _, m := range manifests { + if m.Key == key { + // Compare server-side computed checksum of uploaded file + // against the client's manifest entry (assuming client's + // manifest has correct checksums - client doesn't compute + // checksum of the file before uploading). + + // Try matching sha1 first (newer versions have moved to sha256). + if strings.ToUpper(sum.Algorithm) == "SHA1" { + if sum.Checksum != m.Sha1 { + msg := fmt.Sprintf("manifest checksum %v mismatch with uploaded checksum %v for file %v", + sum.Checksum, m.Sha1, file) + return errors.New(msg) + } + // Uploaded file checksum computed by server matches with local manifest entry. + return nil + } + // If not sha1, check for other types (in a separate field). + if !strings.EqualFold(sum.Algorithm, m.ChecksumType) { + msg := fmt.Sprintf("manifest checksum type %v mismatch with uploaded checksum type %v for file %v", + sum.Algorithm, m.ChecksumType, file) + return errors.New(msg) + } + if !strings.EqualFold(sum.Checksum, m.Checksum) { + msg := fmt.Sprintf("manifest checksum %v mismatch with uploaded checksum %v for file %v", + sum.Checksum, m.Checksum, file) + return errors.New(msg) + } + // Uploaded file checksum computed by server matches with local manifest entry. + return nil + } + } + msg := fmt.Sprintf("missing manifest entry on server for uploaded file %v (key %v), manifests=%#v", file, key, manifests) + return errors.New(msg) +} + +func (imp *Importer) Upload(ctx context.Context, lease *nfc.Lease, item nfc.FileItem) error { + file := item.Path + + f, size, err := imp.Archive.Open(file) + if err != nil { + return err + } + defer f.Close() + + logger := progress.NewProgressLogger(imp.Log, fmt.Sprintf("Uploading %s... ", path.Base(file))) + defer logger.Wait() + + opts := soap.Upload{ + ContentLength: size, + Progress: logger, + } + + err = lease.Upload(ctx, item, f, opts) + if err != nil { + return err + } + + if imp.VerifyManifest { + mapImportKeyToKey := func(urls []types.HttpNfcLeaseDeviceUrl, importKey string) string { + for _, url := range urls { + if url.ImportKey == importKey { + return url.Key + } + } + return "" + } + leaseInfo, err := lease.Wait(ctx, nil) + if err != nil { + return err + } + sum, ok := imp.Manifest[file] + if !ok { + return fmt.Errorf("missing checksum for %v in manifest file", file) + } + return ValidateChecksum(ctx, lease, sum, file, mapImportKeyToKey(leaseInfo.DeviceUrl, item.DeviceId)) + } + return nil +} diff --git a/ovf/importer/options.go b/ovf/importer/options.go new file mode 100644 index 000000000..cbb00af62 --- /dev/null +++ b/ovf/importer/options.go @@ -0,0 +1,91 @@ +/* +Copyright (c) 2024-2024 VMware, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package importer + +import ( + "encoding/json" + + "github.com/vmware/govmomi/ovf" + "github.com/vmware/govmomi/vim25/types" +) + +type KeyValue struct { + Key string + Value string +} + +// case insensitive for Key + Value +func (kv *KeyValue) UnmarshalJSON(b []byte) error { + e := struct { + types.KeyValue + Key *string + Value *string + }{ + types.KeyValue{}, &kv.Key, &kv.Value, + } + + err := json.Unmarshal(b, &e) + if err != nil { + return err + } + + if kv.Key == "" { + kv.Key = e.KeyValue.Key // "key" + } + + if kv.Value == "" { + kv.Value = e.KeyValue.Value // "value" + } + + return nil +} + +type Property struct { + KeyValue + Spec *ovf.Property `json:",omitempty"` +} + +type Network struct { + Name string + Network string +} + +type Options struct { + AllDeploymentOptions []string `json:",omitempty"` + Deployment string `json:",omitempty"` + + AllDiskProvisioningOptions []string `json:",omitempty"` + DiskProvisioning string + + AllIPAllocationPolicyOptions []string `json:",omitempty"` + IPAllocationPolicy string + + AllIPProtocolOptions []string `json:",omitempty"` + IPProtocol string + + PropertyMapping []Property `json:",omitempty"` + + NetworkMapping []Network `json:",omitempty"` + + Annotation string `json:",omitempty"` + + MarkAsTemplate bool + PowerOn bool + InjectOvfEnv bool + WaitForIP bool + Name *string +} diff --git a/govc/importx/options_test.go b/ovf/importer/options_test.go similarity index 90% rename from govc/importx/options_test.go rename to ovf/importer/options_test.go index ac6e35bab..d447215cb 100644 --- a/govc/importx/options_test.go +++ b/ovf/importer/options_test.go @@ -1,5 +1,5 @@ /* -Copyright (c) 2023-2023 VMware, Inc. All Rights Reserved. +Copyright (c) 2024-2024 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,14 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package importx_test +package importer_test import ( "bytes" "encoding/json" "testing" - "github.com/vmware/govmomi/govc/importx" + "github.com/vmware/govmomi/ovf/importer" ) func TestDecodeOptions(t *testing.T) { @@ -52,7 +52,7 @@ func TestDecodeOptions(t *testing.T) { "Name": null } `) - var opts importx.Options + var opts importer.Options err := json.NewDecoder(bytes.NewReader(spec)).Decode(&opts) if err != nil { t.Fatal(err) diff --git a/ovf/importer/spec.go b/ovf/importer/spec.go new file mode 100644 index 000000000..a0028ec78 --- /dev/null +++ b/ovf/importer/spec.go @@ -0,0 +1,138 @@ +/* +Copyright (c) 2024-2024 VMware, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package importer + +import ( + "golang.org/x/text/cases" + "golang.org/x/text/language" + + "github.com/vmware/govmomi/ovf" + "github.com/vmware/govmomi/vim25/types" +) + +var ( + allDiskProvisioningOptions = types.OvfCreateImportSpecParamsDiskProvisioningType("").Strings() + + allIPAllocationPolicyOptions = types.VAppIPAssignmentInfoIpAllocationPolicy("").Strings() + + allIPProtocolOptions = types.VAppIPAssignmentInfoProtocols("").Strings() +) + +func SpecMap(e *ovf.Envelope, hidden, verbose bool) (res []Property) { + if e == nil || e.VirtualSystem == nil { + return nil + } + + for _, p := range e.VirtualSystem.Product { + for i, v := range p.Property { + if v.UserConfigurable == nil { + continue + } + if !*v.UserConfigurable && !hidden { + continue + } + + d := "" + if v.Default != nil { + d = *v.Default + } + + // vSphere only accept True/False as boolean values for some reason + if v.Type == "boolean" { + d = cases.Title(language.Und).String(d) + } + + np := Property{KeyValue: KeyValue{Key: p.Key(v), Value: d}} + + if verbose { + np.Spec = &p.Property[i] + } + + res = append(res, np) + } + } + + return +} + +func Spec(fpath string, a Archive, hidden, verbose bool) (*Options, error) { + e := &ovf.Envelope{} + if fpath != "" { + d, err := ReadOvf(fpath, a) + if err != nil { + return nil, err + } + + if e, err = ReadEnvelope(d); err != nil { + return nil, err + } + } + + var deploymentOptions []string + if e.DeploymentOption != nil && e.DeploymentOption.Configuration != nil { + // add default first + for _, c := range e.DeploymentOption.Configuration { + if c.Default != nil && *c.Default { + deploymentOptions = append(deploymentOptions, c.ID) + } + } + + for _, c := range e.DeploymentOption.Configuration { + if c.Default == nil || !*c.Default { + deploymentOptions = append(deploymentOptions, c.ID) + } + } + } + + o := Options{ + DiskProvisioning: allDiskProvisioningOptions[0], + IPAllocationPolicy: allIPAllocationPolicyOptions[0], + IPProtocol: allIPProtocolOptions[0], + MarkAsTemplate: false, + PowerOn: false, + WaitForIP: false, + InjectOvfEnv: false, + PropertyMapping: SpecMap(e, hidden, verbose), + } + + if deploymentOptions != nil { + o.Deployment = deploymentOptions[0] + } + + if e.VirtualSystem != nil && e.VirtualSystem.Annotation != nil { + for _, a := range e.VirtualSystem.Annotation { + o.Annotation += a.Annotation + } + } + + if e.Network != nil { + for _, net := range e.Network.Networks { + o.NetworkMapping = append(o.NetworkMapping, Network{net.Name, ""}) + } + } + + if verbose { + if deploymentOptions != nil { + o.AllDeploymentOptions = deploymentOptions + } + o.AllDiskProvisioningOptions = allDiskProvisioningOptions + o.AllIPAllocationPolicyOptions = allIPAllocationPolicyOptions + o.AllIPProtocolOptions = allIPProtocolOptions + } + + return &o, nil +} diff --git a/scripts/license.sh b/scripts/license.sh index 7bb1302cc..a9b59e1ce 100755 --- a/scripts/license.sh +++ b/scripts/license.sh @@ -7,7 +7,7 @@ header_dir=$(dirname $0)/headers tmpfile=$(mktemp) trap "rm -f ${tmpfile}" EXIT -git diff --name-status main | awk '{print $2}' | while read file; do +git diff --name-status main | awk '{print $NF}' | while read file; do years=( $(git log --format='%ai' $file | cut -d- -f1 | sort -u) ) num_years=${#years[@]} diff --git a/vim25/progress/loger.go b/vim25/progress/loger.go new file mode 100644 index 000000000..b1356688f --- /dev/null +++ b/vim25/progress/loger.go @@ -0,0 +1,125 @@ +/* +Copyright (c) 2024-2024 VMware, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package progress + +import ( + "fmt" + "io" + "sync" + "time" +) + +type LogFunc func(msg string) (int, error) + +type ProgressLogger struct { + log LogFunc + prefix string + + wg sync.WaitGroup + + sink chan chan Report + done chan struct{} +} + +func NewProgressLogger(log LogFunc, prefix string) *ProgressLogger { + p := &ProgressLogger{ + log: log, + prefix: prefix, + + sink: make(chan chan Report), + done: make(chan struct{}), + } + + p.wg.Add(1) + + go p.loopA() + + return p +} + +// loopA runs before Sink() has been called. +func (p *ProgressLogger) loopA() { + var err error + + defer p.wg.Done() + + tick := time.NewTicker(100 * time.Millisecond) + defer tick.Stop() + + called := false + + for stop := false; !stop; { + select { + case ch := <-p.sink: + err = p.loopB(tick, ch) + stop = true + called = true + case <-p.done: + stop = true + case <-tick.C: + line := fmt.Sprintf("\r%s", p.prefix) + p.log(line) + } + } + + if err != nil && err != io.EOF { + p.log(fmt.Sprintf("\r%sError: %s\n", p.prefix, err)) + } else if called { + p.log(fmt.Sprintf("\r%sOK\n", p.prefix)) + } +} + +// loopA runs after Sink() has been called. +func (p *ProgressLogger) loopB(tick *time.Ticker, ch <-chan Report) error { + var r Report + var ok bool + var err error + + for ok = true; ok; { + select { + case r, ok = <-ch: + if !ok { + break + } + err = r.Error() + case <-tick.C: + line := fmt.Sprintf("\r%s", p.prefix) + if r != nil { + line += fmt.Sprintf("(%.0f%%", r.Percentage()) + detail := r.Detail() + if detail != "" { + line += fmt.Sprintf(", %s", detail) + } + line += ")" + } + p.log(line) + } + } + + return err +} + +func (p *ProgressLogger) Sink() chan<- Report { + ch := make(chan Report) + p.sink <- ch + return ch +} + +func (p *ProgressLogger) Wait() { + close(p.done) + p.wg.Wait() +}