From a0bbbf3f563e095bc0c19d21170172a28355392a Mon Sep 17 00:00:00 2001 From: Doug MacEachern Date: Fri, 31 May 2024 14:38:28 -0700 Subject: [PATCH] vcsim: add library item storage API support govc: support Content Library iso files for cdrom backing The 'device.cdrom.insert' and 'vm.create -iso' commands can now a URL with 'library' scheme to back a cdrom device from a Content Library. api: refactor 'library.info -L' functionality into library PathFinder helper. Fixes #3213 --- govc/USAGE.md | 8 +- govc/device/cdrom/insert.go | 9 +- govc/flags/datastore.go | 63 ++++++++- govc/library/info.go | 171 ++++++------------------ govc/test/library.bats | 45 +++++-- govc/vm/create.go | 12 +- vapi/library/finder/path.go | 132 +++++++++++++++++++ vapi/library/library_item_storage.go | 8 +- vapi/simulator/simulator.go | 190 ++++++++++++++++++++++----- 9 files changed, 449 insertions(+), 189 deletions(-) create mode 100644 vapi/library/finder/path.go diff --git a/govc/USAGE.md b/govc/USAGE.md index c647cfa49..88d981e50 100644 --- a/govc/USAGE.md +++ b/govc/USAGE.md @@ -1553,6 +1553,7 @@ If device is not specified, the first CD-ROM device is used. Examples: govc device.cdrom.insert -vm vm-1 -device cdrom-3000 images/boot.iso + govc device.cdrom.insert -vm vm-1 library:/boot/linux/ubuntu.iso # Content Library ISO Options: -device= CD-ROM device name @@ -3631,6 +3632,8 @@ Usage: govc library.info [OPTIONS] Display library information. +Note: the '-s' flag only applies to files, not items or the library itself. + Examples: govc library.info govc library.info /lib1 @@ -3638,7 +3641,8 @@ Examples: govc library.info /lib1/item1 govc library.info /lib1/item1/ govc library.info */ - govc device.cdrom.insert -vm $vm -device cdrom-3000 $(govc library.info -L /lib1/item1/file1) + govc library.info -L /lib1/item1/file1 # file path relative to Datastore + govc library.info -L -l /lib1/item1/file1 # file path including Datastore govc library.info -json | jq . govc library.info -json /lib1/item1 | jq . @@ -3646,6 +3650,7 @@ Options: -L=false List Datastore path only -U=false List pub/sub URL(s) only -l=false Long listing format + -s=false Include file specific storage details ``` ## library.ls @@ -6284,6 +6289,7 @@ https://code.vmware.com/apis/358/vsphere/doc/vim.vm.GuestOsDescriptor.GuestOsIde Examples: govc vm.create -on=false vm-name + govc vm.create -iso library:/boot/linux/ubuntu.iso vm-name # Content Library ISO govc vm.create -cluster cluster1 vm-name # use compute cluster placement govc vm.create -datastore-cluster dscluster vm-name # use datastore cluster placement govc vm.create -m 2048 -c 2 -g freebsd64Guest -net.adapter vmxnet3 -disk.controller pvscsi vm-name diff --git a/govc/device/cdrom/insert.go b/govc/device/cdrom/insert.go index 73f3ce68d..dc38a6034 100644 --- a/govc/device/cdrom/insert.go +++ b/govc/device/cdrom/insert.go @@ -1,11 +1,11 @@ /* -Copyright (c) 2014-2016 VMware, Inc. All Rights Reserved. +Copyright (c) 2014-2024 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -64,7 +64,8 @@ func (cmd *insert) Description() string { If device is not specified, the first CD-ROM device is used. Examples: - govc device.cdrom.insert -vm vm-1 -device cdrom-3000 images/boot.iso` + govc device.cdrom.insert -vm vm-1 -device cdrom-3000 images/boot.iso + govc device.cdrom.insert -vm vm-1 library:/boot/linux/ubuntu.iso # Content Library ISO` } func (cmd *insert) Run(ctx context.Context, f *flag.FlagSet) error { @@ -87,7 +88,7 @@ func (cmd *insert) Run(ctx context.Context, f *flag.FlagSet) error { return err } - iso, err := cmd.DatastorePath(f.Arg(0)) + iso, err := cmd.FileBacking(ctx, f.Arg(0), false) if err != nil { return err } diff --git a/govc/flags/datastore.go b/govc/flags/datastore.go index 8030a6e00..7ca7c1a71 100644 --- a/govc/flags/datastore.go +++ b/govc/flags/datastore.go @@ -1,11 +1,11 @@ /* -Copyright (c) 2014-2016 VMware, Inc. All Rights Reserved. +Copyright (c) 2014-2024 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -20,9 +20,12 @@ import ( "context" "flag" "fmt" + "net/url" "os" "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vapi/library" + "github.com/vmware/govmomi/vapi/library/finder" "github.com/vmware/govmomi/vim25/types" ) @@ -148,3 +151,59 @@ func (f *DatastoreFlag) Stat(ctx context.Context, file string) (types.BaseFileIn return ds.Stat(ctx, file) } + +func (f *DatastoreFlag) libraryPath(ctx context.Context, p string) (string, error) { + vc, err := f.Client() + if err != nil { + return "", err + } + + rc, err := f.RestClient() + if err != nil { + return "", err + } + + m := library.NewManager(rc) + + r, err := finder.NewFinder(m).Find(ctx, p) + if err != nil { + return "", err + } + + if len(r) != 1 { + return "", fmt.Errorf("%s: %d found", p, len(r)) + } + + return finder.NewPathFinder(m, vc).Path(ctx, r[0]) +} + +// FileBacking converts the given file path for use as VirtualDeviceFileBackingInfo.FileName. +func (f *DatastoreFlag) FileBacking(ctx context.Context, file string, stat bool) (string, error) { + u, err := url.Parse(file) + if err != nil { + return "", err + } + + switch u.Scheme { + case "library": + return f.libraryPath(ctx, u.Path) + case "ds": + // datastore url, e.g. ds:///vmfs/volumes/$uuid/... + return file, nil + } + + var p object.DatastorePath + if p.FromString(file) { + // datastore is specified + return file, nil + } + + if stat { + // Verify ISO exists + if _, err := f.Stat(ctx, file); err != nil { + return "", err + } + } + + return f.DatastorePath(file) +} diff --git a/govc/library/info.go b/govc/library/info.go index 87bd6d073..9b80efc51 100644 --- a/govc/library/info.go +++ b/govc/library/info.go @@ -1,5 +1,5 @@ /* -Copyright (c) 2018-2023 VMware, Inc. All Rights Reserved. +Copyright (c) 2018-2024 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,8 +22,6 @@ import ( "flag" "fmt" "io" - "path" - "strings" "text/tabwriter" "time" @@ -33,10 +31,6 @@ import ( "github.com/vmware/govmomi/units" "github.com/vmware/govmomi/vapi/library" "github.com/vmware/govmomi/vapi/library/finder" - - "github.com/vmware/govmomi/property" - "github.com/vmware/govmomi/vim25/mo" - "github.com/vmware/govmomi/vim25/types" ) type info struct { @@ -47,8 +41,9 @@ type info struct { long bool link bool url bool + stor bool - names map[string]string + pathFinder *finder.PathFinder } func init() { @@ -66,8 +61,7 @@ func (cmd *info) Register(ctx context.Context, f *flag.FlagSet) { f.BoolVar(&cmd.long, "l", false, "Long listing format") f.BoolVar(&cmd.link, "L", false, "List Datastore path only") f.BoolVar(&cmd.url, "U", false, "List pub/sub URL(s) only") - - cmd.names = make(map[string]string) + f.BoolVar(&cmd.stor, "s", false, "Include file specific storage details") } func (cmd *info) Process(ctx context.Context) error { @@ -80,6 +74,8 @@ func (cmd *info) Process(ctx context.Context) error { func (cmd *info) Description() string { return `Display library information. +Note: the '-s' flag only applies to files, not items or the library itself. + Examples: govc library.info govc library.info /lib1 @@ -87,7 +83,8 @@ Examples: govc library.info /lib1/item1 govc library.info /lib1/item1/ govc library.info */ - govc device.cdrom.insert -vm $vm -device cdrom-3000 $(govc library.info -L /lib1/item1/file1) + govc library.info -L /lib1/item1/file1 # file path relative to Datastore + govc library.info -L -l /lib1/item1/file1 # file path including Datastore govc library.info -json | jq . govc library.info -json /lib1/item1 | jq .` } @@ -96,6 +93,7 @@ type infoResultsWriter struct { Result []finder.FindResult `json:"result"` m *library.Manager cmd *info + ctx context.Context } func (r infoResultsWriter) MarshalJSON() ([]byte, error) { @@ -113,7 +111,7 @@ func (r infoResultsWriter) Dump() interface{} { func (r infoResultsWriter) Write(w io.Writer) error { if r.cmd.link { for _, j := range r.Result { - p, err := r.cmd.getDatastoreFilePath(j) + p, err := r.cmd.pathFinder.Path(context.Background(), j) if err != nil { return err } @@ -180,8 +178,12 @@ func (r infoResultsWriter) writeLibrary( fmt.Fprintf(w, " Type:\t%s\n", d.Type) } if r.cmd.long { - fmt.Fprintf(w, " Datastore Path:\t%s\n", r.cmd.getDatastorePath(res)) - items, err := r.m.GetLibraryItems(context.Background(), v.ID) + p, err := r.cmd.pathFinder.Path(r.ctx, res) + if err != nil { + return err + } + fmt.Fprintf(w, " Datastore Path:\t%s\n", p) + items, err := r.m.GetLibraryItems(r.ctx, v.ID) if err != nil { return err } @@ -234,7 +236,11 @@ func (r infoResultsWriter) writeItem( fmt.Fprintf(w, " Certificate Status:\t%s\n", v.CertificateVerification.Status) } if r.cmd.long { - fmt.Fprintf(w, " Datastore Path:\t%s\n", r.cmd.getDatastorePath(res)) + p, err := r.cmd.pathFinder.Path(r.ctx, res) + if err != nil { + return err + } + fmt.Fprintf(w, " Datastore Path:\t%s\n", p) } return nil @@ -253,7 +259,22 @@ func (r infoResultsWriter) writeFile( fmt.Fprintf(w, " Version:\t%s\n", v.Version) if r.cmd.long { - fmt.Fprintf(w, " Datastore Path:\t%s\n", r.cmd.getDatastorePath(res)) + p, err := r.cmd.pathFinder.Path(r.ctx, res) + if err != nil { + return err + } + fmt.Fprintf(w, " Datastore Path:\t%s\n", p) + } + if r.cmd.stor { + s, err := r.m.GetLibraryItemStorage(r.ctx, res.GetParent().GetID(), v.Name) + if err != nil { + return err + } + for i := range s { + for _, uri := range s[i].StorageURIs { + fmt.Fprintf(w, " Storage URI:\t%s\n", uri) + } + } } return nil @@ -264,124 +285,18 @@ func (cmd *info) Run(ctx context.Context, f *flag.FlagSet) error { if err != nil { return err } + vc, err := cmd.Client() + if err != nil { + return err + } m := library.NewManager(c) + cmd.pathFinder = finder.NewPathFinder(m, vc) finder := finder.NewFinder(m) findResults, err := finder.Find(ctx, f.Args()...) if err != nil { return err } - // Lookup the names(s) of the library's datastore(s). - for i := range findResults { - if t, ok := findResults[i].GetResult().(library.Library); ok { - for j := range t.Storage { - if t.Storage[j].Type == "DATASTORE" { - t.Storage[j].DatastoreID = cmd.getDatastoreName(t.Storage[j].DatastoreID) - } - } - } - } - return cmd.WriteResult(&infoResultsWriter{findResults, m, cmd}) -} - -func (cmd *info) getDatastoreName(id string) string { - if name, ok := cmd.names[id]; ok { - return name - } - - c, err := cmd.Client() - if err != nil { - return id - } - - obj := types.ManagedObjectReference{ - Type: "Datastore", - Value: id, - } - pc := property.DefaultCollector(c) - var me mo.ManagedEntity - - err = pc.RetrieveOne(context.Background(), obj, []string{"name"}, &me) - if err != nil { - return id - } - - cmd.names[id] = me.Name - return me.Name -} - -func (cmd *info) getDatastorePath(r finder.FindResult) string { - p, _ := cmd.getDatastoreFilePath(r) - return p -} - -func (cmd *info) getDatastoreFilePath(r finder.FindResult) (string, error) { - switch x := r.GetResult().(type) { - case library.Library: - id := "" - if len(x.Storage) != 0 { - id = cmd.getDatastoreName(x.Storage[0].DatastoreID) - } - return fmt.Sprintf("[%s] contentlib-%s", id, x.ID), nil - case library.Item: - return fmt.Sprintf("%s/%s", cmd.getDatastorePath(r.GetParent()), x.ID), nil - case library.File: - return cmd.getDatastoreFileItemPath(r) - default: - return "", fmt.Errorf("unsupported type=%T", x) - } -} - -// getDatastoreFileItemPath returns the absolute datastore path for a library.File -func (cmd *info) getDatastoreFileItemPath(r finder.FindResult) (string, error) { - name := r.GetName() - dir := cmd.getDatastorePath(r.GetParent()) - p := path.Join(dir, name) - - lib := r.GetParent().GetParent().GetResult().(library.Library) - if len(lib.Storage) == 0 { - return p, nil - } - - ctx := context.Background() - c, err := cmd.Client() - if err != nil { - return p, err - } - - ref := types.ManagedObjectReference{Type: "Datastore", Value: lib.Storage[0].DatastoreID} - ds := object.NewDatastore(c, ref) - - b, err := ds.Browser(ctx) - if err != nil { - return p, err - } - - // The file ID isn't available via the API, so we use DatastoreBrowser to search - ext := path.Ext(name) - pat := strings.Replace(name, ext, "*"+ext, 1) - spec := types.HostDatastoreBrowserSearchSpec{ - MatchPattern: []string{pat}, - } - - task, err := b.SearchDatastore(ctx, dir, &spec) - if err != nil { - return p, err - } - - info, err := task.WaitForResult(ctx, nil) - if err != nil { - return p, err - } - - res, ok := info.Result.(types.HostDatastoreBrowserSearchResults) - if !ok { - return p, fmt.Errorf("search(%s) result type=%T", pat, info.Result) - } - - if len(res.File) != 1 { - return p, fmt.Errorf("search(%s) result files=%d", pat, len(res.File)) - } - return path.Join(dir, res.File[0].GetFileInfo().Path), nil + return cmd.WriteResult(&infoResultsWriter{findResults, m, cmd, ctx}) } diff --git a/govc/test/library.bats b/govc/test/library.bats index 058868187..5ae05f294 100755 --- a/govc/test/library.bats +++ b/govc/test/library.bats @@ -64,10 +64,6 @@ load test_helper @test "library.import" { vcsim_env - run govc session.ls - assert_success - govc session.ls -json | jq . - run govc library.create my-content assert_success library_id="$output" @@ -171,9 +167,24 @@ load test_helper assert_matches "$TTYLINUX_NAME.ovf" assert_matches "$TTYLINUX_NAME-disk1.vmdk" - run govc session.ls + summary="ISO \[${GOVC_DATASTORE}\] contentlib-.*${TTYLINUX_NAME}.iso" + + run govc vm.create -on=false -iso "library:/my-content/ttylinux-live/$TTYLINUX_NAME.iso" library-iso-test + assert_success + + run govc device.info -vm library-iso-test cdrom-* assert_success - govc session.ls -json | jq . + assert_matches "$summary" + + run govc device.cdrom.eject -vm library-iso-test + assert_success + + run govc device.cdrom.insert -vm library-iso-test "library:/my-content/ttylinux-live/$TTYLINUX_NAME.iso" + assert_success + + run govc device.info -vm library-iso-test cdrom-* + assert_success + assert_matches "$summary" } @test "library.deploy" { @@ -187,8 +198,17 @@ load test_helper dir=$(govc datastore.info -json | jq -r .datastores[].info.url) ln -s "$GOVC_IMAGES/$TTYLINUX_NAME."* "$dir" - # vcsim doesn't verify checksums. Use a fake checksum and a possible algorithm to ensure the args are accepted. - run govc library.import -pull -c=fake -a=SHA1 my-content "https://$(govc env GOVC_URL)/folder/$TTYLINUX_NAME.ovf" + run govc library.import -c fake -pull my-content -n invalid-sha1 "https://$(govc env GOVC_URL)/folder/$TTYLINUX_NAME.ovf" + assert_failure # invalid checksum + + sum=$(sha256sum "$GOVC_IMAGES/$TTYLINUX_NAME.ovf" | awk '{print $1}') + run govc library.import -c "$sum" -pull my-content "https://$(govc env GOVC_URL)/folder/$TTYLINUX_NAME.ovf" + assert_success + + run govc library.info -s "/my-content/ttylinux-live/$TTYLINUX_NAME.ovf" + assert_success + + run govc library.info -l -s /my-content/$TTYLINUX_NAME/$TTYLINUX_NAME.ovf assert_success run govc library.deploy "my-content/$TTYLINUX_NAME" ttylinux @@ -197,8 +217,11 @@ load test_helper run govc vm.info ttylinux assert_success - # vcsim doesn't verify checksums. Use a fake checksum and a possible algorithm to ensure the args are accepted. - run govc library.import -pull -c=fake -a=MD5 -n ttylinux-unpacked my-content "https://$(govc env GOVC_URL)/folder/$TTYLINUX_NAME.ova" + run govc library.import -pull -c fake -a MD5 -n invalid-md5 my-content "https://$(govc env GOVC_URL)/folder/$TTYLINUX_NAME.ova" + assert_failure # invalid checksum + + sum=$(md5sum "$GOVC_IMAGES/$TTYLINUX_NAME.ovf" | awk '{print $1}') + run govc library.import -pull -c "$sum" -a MD5 -n ttylinux-unpacked my-content "https://$(govc env GOVC_URL)/folder/$TTYLINUX_NAME.ova" assert_success item_id=$(govc library.info -json /my-content/ttylinux-unpacked | jq -r .[].id) @@ -653,5 +676,3 @@ EOF cached=$(govc library.info subscribed-content/ttylinux-latest | grep Cached: | awk '{print $2}') assert_equal "false" "$cached" } - - diff --git a/govc/vm/create.go b/govc/vm/create.go index f46155431..a2eb6cc8a 100644 --- a/govc/vm/create.go +++ b/govc/vm/create.go @@ -63,7 +63,6 @@ type create struct { iso string isoDatastoreFlag *flags.DatastoreFlag - isoDatastore *object.Datastore disk string diskDatastoreFlag *flags.DatastoreFlag @@ -195,6 +194,7 @@ https://code.vmware.com/apis/358/vsphere/doc/vim.vm.GuestOsDescriptor.GuestOsIde Examples: govc vm.create -on=false vm-name + govc vm.create -iso library:/boot/linux/ubuntu.iso vm-name # Content Library ISO govc vm.create -cluster cluster1 vm-name # use compute cluster placement govc vm.create -datastore-cluster dscluster vm-name # use datastore cluster placement govc vm.create -m 2048 -c 2 -g freebsd64Guest -net.adapter vmxnet3 -disk.controller pvscsi vm-name` @@ -267,15 +267,11 @@ func (cmd *create) Run(ctx context.Context, f *flag.FlagSet) error { // Verify ISO exists if cmd.iso != "" { - _, err = cmd.isoDatastoreFlag.Stat(ctx, cmd.iso) - if err != nil { - return err - } - - cmd.isoDatastore, err = cmd.isoDatastoreFlag.Datastore() + iso, err := cmd.isoDatastoreFlag.FileBacking(ctx, cmd.iso, true) if err != nil { return err } + cmd.iso = iso } // Verify disk exists @@ -505,7 +501,7 @@ func (cmd *create) addStorage(devices object.VirtualDeviceList) (object.VirtualD return nil, err } - cdrom = devices.InsertIso(cdrom, cmd.isoDatastore.Path(cmd.iso)) + cdrom = devices.InsertIso(cdrom, cmd.iso) devices = append(devices, cdrom) } diff --git a/vapi/library/finder/path.go b/vapi/library/finder/path.go new file mode 100644 index 000000000..213e1ebd3 --- /dev/null +++ b/vapi/library/finder/path.go @@ -0,0 +1,132 @@ +/* +Copyright (c) 2024-2024 VMware, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package finder + +import ( + "context" + "fmt" + "net/url" + "path" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vapi/library" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" +) + +// PathFinder is used to find the Datastore path of a library.Library, library.Item or library.File. +type PathFinder struct { + m *library.Manager + c *vim25.Client + cache map[string]string +} + +// NewPathFinder creates a new PathFinder instance. +func NewPathFinder(m *library.Manager, c *vim25.Client) *PathFinder { + return &PathFinder{ + m: m, + c: c, + cache: make(map[string]string), + } +} + +// Path returns the absolute datastore path for a Library, Item or File. +// The cache is used by DatastoreName(). +func (f *PathFinder) Path(ctx context.Context, r FindResult) (string, error) { + switch l := r.GetResult().(type) { + case library.Library: + id := "" + if len(l.Storage) != 0 { + var err error + id, err = f.datastoreName(ctx, l.Storage[0].DatastoreID) + if err != nil { + return "", err + } + } + return fmt.Sprintf("[%s] contentlib-%s", id, l.ID), nil + case library.Item: + p, err := f.Path(ctx, r.GetParent()) + if err != nil { + return "", err + } + return fmt.Sprintf("%s/%s", p, l.ID), nil + case library.File: + return f.getFileItemPath(ctx, r) + default: + return "", fmt.Errorf("unsupported type=%T", l) + } +} + +// getFileItemPath returns the absolute datastore path for a library.File +func (f *PathFinder) getFileItemPath(ctx context.Context, r FindResult) (string, error) { + name := r.GetName() + + dir, err := f.Path(ctx, r.GetParent()) + if err != nil { + return "", err + } + + p := path.Join(dir, name) + + lib := r.GetParent().GetParent().GetResult().(library.Library) + if len(lib.Storage) == 0 { + return p, nil + } + + // storage file name includes a uuid, for example: + // "ubuntu-14.04.6-server-amd64.iso" -> "ubuntu-14.04.6-server-amd64_0653e3f3-b4f4-41fb-9b72-c4102450e3dc.iso" + s, err := f.m.GetLibraryItemStorage(ctx, r.GetParent().GetID(), name) + if err != nil { + return p, err + } + // Currently there can only be 1 storage URI + if len(s) == 0 { + return p, nil + } + + uris := s[0].StorageURIs + if len(uris) == 0 { + return p, nil + } + u, err := url.Parse(uris[0]) + if err != nil { + return p, err + } + + return path.Join(dir, path.Base(u.Path)), nil +} + +// datastoreName returns the Datastore.Name for the given id. +func (f *PathFinder) datastoreName(ctx context.Context, id string) (string, error) { + if name, ok := f.cache[id]; ok { + return name, nil + } + + obj := types.ManagedObjectReference{ + Type: "Datastore", + Value: id, + } + + ds := object.NewDatastore(f.c, obj) + name, err := ds.ObjectName(ctx) + if err != nil { + return "", err + } + + f.cache[id] = name + return name, nil +} diff --git a/vapi/library/library_item_storage.go b/vapi/library/library_item_storage.go index 9ac60e492..4e9fe9156 100644 --- a/vapi/library/library_item_storage.go +++ b/vapi/library/library_item_storage.go @@ -43,11 +43,11 @@ func (c *Manager) ListLibraryItemStorage(ctx context.Context, id string) ([]Stor } // GetLibraryItemStorage returns the storage for a specific file in a library item. -func (c *Manager) GetLibraryItemStorage(ctx context.Context, id, fileName string) (*Storage, error) { +func (c *Manager) GetLibraryItemStorage(ctx context.Context, id, fileName string) ([]Storage, error) { url := c.Resource(internal.LibraryItemStoragePath).WithID(id).WithAction("get") spec := struct { - Name string `json:"name"` + Name string `json:"file_name"` }{fileName} - var res Storage - return &res, c.Do(ctx, url.Request(http.MethodPost, spec), &res) + var res []Storage + return res, c.Do(ctx, url.Request(http.MethodPost, spec), &res) } diff --git a/vapi/simulator/simulator.go b/vapi/simulator/simulator.go index b1d23189f..ec571b948 100644 --- a/vapi/simulator/simulator.go +++ b/vapi/simulator/simulator.go @@ -1,5 +1,5 @@ /* -Copyright (c) 2018-2023 VMware, Inc. All Rights Reserved. +Copyright (c) 2018-2024 VMware, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,6 +20,10 @@ import ( "archive/tar" "bytes" "context" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" "crypto/tls" "crypto/x509" "encoding/base64" @@ -27,6 +31,7 @@ import ( "encoding/pem" "errors" "fmt" + "hash" "io" "log" "net/http" @@ -75,6 +80,7 @@ type content struct { } type update struct { + *sync.WaitGroup *library.Session Library *library.Library File map[string]*library.UpdateFile @@ -152,6 +158,8 @@ func New(u *url.URL, r *simulator.Registry) ([]string, http.Handler) { {internal.Subscriptions + "/", s.subscriptionsID}, {internal.LibraryItemPath, s.libraryItem}, {internal.LibraryItemPath + "/", s.libraryItemID}, + {internal.LibraryItemStoragePath, s.libraryItemStorage}, + {internal.LibraryItemStoragePath + "/", s.libraryItemStorageID}, {internal.SubscribedLibraryItem + "/", s.libraryItemID}, {internal.LibraryItemUpdateSession, s.libraryItemUpdateSession}, {internal.LibraryItemUpdateSession + "/", s.libraryItemUpdateSessionID}, @@ -1282,6 +1290,93 @@ func (s *handler) libraryItemID(w http.ResponseWriter, r *http.Request) { } } +func (s *handler) libraryItemByID(id string) (*content, *item) { + for _, l := range s.Library { + if item, ok := l.Item[id]; ok { + return l, item + } + } + + log.Printf("library for item %q not found", id) + + return nil, nil +} + +func (s *handler) libraryItemStorageByID(id string) ([]library.Storage, bool) { + lib, item := s.libraryItemByID(id) + if item == nil { + return nil, false + } + + storage := make([]library.Storage, len(item.File)) + + for i, file := range item.File { + storage[i] = library.Storage{ + StorageBacking: lib.Storage[0], + StorageURIs: []string{ + path.Join(libraryPath(lib.Library, id), file.Name), + }, + Name: file.Name, + Version: file.Version, + } + if file.Checksum != nil { + storage[i].Checksum = *file.Checksum + } + if file.Size != nil { + storage[i].Size = *file.Size + } + if file.Cached != nil { + storage[i].Cached = *file.Cached + } + } + + return storage, true +} + +func (s *handler) libraryItemStorage(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + id := r.URL.Query().Get("library_item_id") + storage, ok := s.libraryItemStorageByID(id) + if !ok { + http.NotFound(w, r) + return + } + + OK(w, storage) +} + +func (s *handler) libraryItemStorageID(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + id := s.id(r) + storage, ok := s.libraryItemStorageByID(id) + if !ok { + http.NotFound(w, r) + return + } + + var spec struct { + Name string `json:"file_name"` + } + + if s.decode(r, w, &spec) { + for _, file := range storage { + if file.Name == spec.Name { + OK(w, []library.Storage{file}) + return + } + } + http.NotFound(w, r) + } +} + func (s *handler) libraryItemUpdateSession(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodGet: @@ -1315,9 +1410,10 @@ func (s *handler) libraryItemUpdateSession(w http.ResponseWriter, r *http.Reques ExpirationTime: types.NewTime(time.Now().Add(time.Hour)), } s.Update[session.ID] = update{ - Session: session, - Library: lib, - File: make(map[string]*library.UpdateFile), + WaitGroup: new(sync.WaitGroup), + Session: session, + Library: lib, + File: make(map[string]*library.UpdateFile), } OK(w, session.ID) } @@ -1335,7 +1431,9 @@ func (s *handler) libraryItemUpdateSessionID(w http.ResponseWriter, r *http.Requ session := up.Session done := func(state string) { - up.State = state + if up.State != "ERROR" { + up.State = state + } go time.AfterFunc(session.ExpirationTime.Sub(time.Now()), func() { s.Lock() delete(s.Update, id) @@ -1351,7 +1449,10 @@ func (s *handler) libraryItemUpdateSessionID(w http.ResponseWriter, r *http.Requ case "cancel": done("CANCELED") case "complete": - done("DONE") + go func() { + up.Wait() // wait for any PULL sources to complete + done("DONE") + }() case "fail": done("ERROR") case "keep-alive": @@ -1466,14 +1567,16 @@ func (s *handler) libraryItemUpdateSessionFile(w http.ResponseWriter, r *http.Re } func (s *handler) pullSource(up update, info *library.UpdateFile) { + defer up.Done() done := func(err error) { s.Lock() info.Status = "READY" if err != nil { log.Printf("PULL %s: %s", info.SourceEndpoint.URI, err) info.Status = "ERROR" - up.State = "ERROR" - up.ErrorMessage = &rest.LocalizableMessage{DefaultMessage: err.Error()} + info.ErrorMessage = &rest.LocalizableMessage{DefaultMessage: err.Error()} + up.State = info.Status + up.ErrorMessage = info.ErrorMessage } s.Unlock() } @@ -1490,10 +1593,21 @@ func (s *handler) pullSource(up update, info *library.UpdateFile) { return } - err = s.libraryItemFileCreate(&up, info.Name, res.Body) + err = s.libraryItemFileCreate(&up, info.Name, res.Body, info.Checksum) done(err) } +func hasChecksum(c *library.Checksum) bool { + return c != nil && c.Checksum != "" +} + +var checksum = map[string]func() hash.Hash{ + "MD5": md5.New, + "SHA1": sha1.New, + "SHA256": sha256.New, + "SHA512": sha512.New, +} + func (s *handler) libraryItemUpdateSessionFileID(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { w.WriteHeader(http.StatusMethodNotAllowed) @@ -1517,6 +1631,7 @@ func (s *handler) libraryItemUpdateSessionFileID(w http.ResponseWriter, r *http. id = uuid.New().String() info := &library.UpdateFile{ Name: spec.File.Name, + Checksum: spec.File.Checksum, SourceType: spec.File.SourceType, Status: "WAITING_FOR_TRANSFER", BytesTransferred: 0, @@ -1530,20 +1645,30 @@ func (s *handler) libraryItemUpdateSessionFileID(w http.ResponseWriter, r *http. } info.UploadEndpoint = &library.TransferEndpoint{URI: u.String()} case "PULL": + if hasChecksum(info.Checksum) && checksum[info.Checksum.Algorithm] == nil { + BadRequest(w, "com.vmware.vapi.std.errors.invalid_argument") + return + } info.SourceEndpoint = spec.File.SourceEndpoint + info.Status = "TRANSFERRING" + up.Add(1) go s.pullSource(up, info) } up.File[id] = info OK(w, info) } case "get": - OK(w, up.Session) - case "list": - var ids []string - for id := range up.File { - ids = append(ids, id) + var spec struct { + File string `json:"file_name"` + } + if s.decode(r, w, &spec) { + for _, f := range up.File { + if f.Name == spec.File { + OK(w, f) + return + } + } } - OK(w, ids) case "remove": if up.State != "ACTIVE" { s.error(w, fmt.Errorf("removeFile not allowed in state %s", up.State)) @@ -1580,20 +1705,12 @@ func (s *handler) libraryItemDownloadSession(w http.ResponseWriter, r *http.Requ switch s.action(r) { case "create", "": - var lib *library.Library - var files []library.File - for _, l := range s.Library { - if item, ok := l.Item[spec.Session.LibraryItemID]; ok { - lib = l.Library - files = item.File - break - } - } - if lib == nil { - log.Printf("library for item %q not found", spec.Session.LibraryItemID) + lib, item := s.libraryItemByID(spec.Session.LibraryItemID) + if item == nil { http.NotFound(w, r) return } + session := &library.Session{ ID: uuid.New().String(), LibraryItemID: spec.Session.LibraryItemID, @@ -1604,10 +1721,10 @@ func (s *handler) libraryItemDownloadSession(w http.ResponseWriter, r *http.Requ } s.Download[session.ID] = download{ Session: session, - Library: lib, + Library: lib.Library, File: make(map[string]*library.DownloadFile), } - for _, file := range files { + for _, file := range item.File { s.Download[session.ID].File[file.Name] = &library.DownloadFile{ Name: file.Name, Status: "UNPREPARED", @@ -1741,7 +1858,7 @@ func libraryPath(l *library.Library, id string) string { return path.Join(append([]string{ds.Info.GetDatastoreInfo().Url, "contentlib-" + l.ID}, id)...) } -func (s *handler) libraryItemFileCreate(up *update, name string, body io.ReadCloser) error { +func (s *handler) libraryItemFileCreate(up *update, name string, body io.ReadCloser, cs *library.Checksum) error { var in io.Reader = body dir := libraryPath(up.Library, up.Session.LibraryItemID) if err := os.MkdirAll(dir, 0750); err != nil { @@ -1770,6 +1887,12 @@ func (s *handler) libraryItemFileCreate(up *update, name string, body io.ReadClo return err } + var h hash.Hash + if hasChecksum(cs) { + h = checksum[cs.Algorithm]() + in = io.TeeReader(in, h) + } + n, err := io.Copy(file, in) _ = body.Close() if err != nil { @@ -1780,6 +1903,13 @@ func (s *handler) libraryItemFileCreate(up *update, name string, body io.ReadClo return err } + if h != nil { + sum := fmt.Sprintf("%x", h.Sum(nil)) + if sum != cs.Checksum { + return fmt.Errorf("checksum mismatch: actual=%s, expected=%s", sum, cs.Checksum) + } + } + i := s.Library[up.Library.ID].Item[up.Session.LibraryItemID] i.File = append(i.File, library.File{ Cached: types.NewBool(true), @@ -1828,7 +1958,7 @@ func (s *handler) libraryItemFileData(w http.ResponseWriter, r *http.Request) { return } - err := s.libraryItemFileCreate(up, name, r.Body) + err := s.libraryItemFileCreate(up, name, r.Body, nil) if err != nil { s.error(w, err) }