From 47310d37446077f9606705a6736952fae1ae5869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Fri, 26 Jul 2024 18:40:07 -0400 Subject: [PATCH 01/16] client: Report source errors too on copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- client/incus_instances.go | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/client/incus_instances.go b/client/incus_instances.go index 308da3ea13e..54af0dae74e 100644 --- a/client/incus_instances.go +++ b/client/incus_instances.go @@ -626,6 +626,9 @@ func (r *ProtocolIncus) tryCreateInstance(req api.InstancesPost, urls []string, operation := req.Source.Operation // Forward targetOp to remote op + chConnect := make(chan error, 1) + chWait := make(chan error, 1) + go func() { success := false var errors []remoteOperationResult @@ -665,13 +668,35 @@ func (r *ProtocolIncus) tryCreateInstance(req api.InstancesPost, urls []string, break } - if !success { - rop.err = remoteOperationError("Failed instance creation", errors) + if success { + chConnect <- nil + close(chConnect) + } else { + chConnect <- remoteOperationError("Failed instance creation", errors) + close(chConnect) + if op != nil { _ = op.Cancel() } } + }() + + if op != nil { + go func() { + chWait <- op.Wait() + close(chWait) + }() + } + + go func() { + var err error + + select { + case err = <-chConnect: + case err = <-chWait: + } + rop.err = err close(rop.chDone) }() From c4c79c817503e0291c12e569fbcf47ae459197e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Fri, 26 Jul 2024 17:57:04 -0400 Subject: [PATCH 02/16] incusd/storage: Have roundVolumeBlockSizeBytes return an error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A VG query failing would cause the function to attempt to divide by 0 causing a Go process panic... Signed-off-by: Stéphane Graber --- internal/server/storage/drivers/driver_common.go | 4 ++-- internal/server/storage/drivers/driver_lvm.go | 10 +++++++--- internal/server/storage/drivers/driver_zfs.go | 4 ++-- .../server/storage/drivers/driver_zfs_volumes.go | 15 ++++++++++++--- internal/server/storage/drivers/interface.go | 2 +- internal/server/storage/drivers/utils.go | 8 ++++++-- internal/server/storage/drivers/volume.go | 5 ++++- 7 files changed, 34 insertions(+), 14 deletions(-) diff --git a/internal/server/storage/drivers/driver_common.go b/internal/server/storage/drivers/driver_common.go index 30f0de44d94..c15de151d85 100644 --- a/internal/server/storage/drivers/driver_common.go +++ b/internal/server/storage/drivers/driver_common.go @@ -525,10 +525,10 @@ func (d *common) DeleteBucketKey(bucket Volume, keyName string, op *operations.O // roundVolumeBlockSizeBytes returns sizeBytes rounded up to the next multiple // of MinBlockBoundary. -func (d *common) roundVolumeBlockSizeBytes(vol Volume, sizeBytes int64) int64 { +func (d *common) roundVolumeBlockSizeBytes(vol Volume, sizeBytes int64) (int64, error) { // QEMU requires image files to be in traditional storage block boundaries. // We use 8k here to ensure our images are compatible with all of our backend drivers. - return roundAbove(MinBlockBoundary, sizeBytes) + return roundAbove(MinBlockBoundary, sizeBytes), nil } func (d *common) isBlockBacked(vol Volume) bool { diff --git a/internal/server/storage/drivers/driver_lvm.go b/internal/server/storage/drivers/driver_lvm.go index a9a16ef3686..3ebf3ea07f0 100644 --- a/internal/server/storage/drivers/driver_lvm.go +++ b/internal/server/storage/drivers/driver_lvm.go @@ -803,8 +803,12 @@ func (d *lvm) GetResources() (*api.ResourcesStoragePool, error) { // roundVolumeBlockSizeBytes returns sizeBytes rounded up to the next multiple // of the volume group extent size. -func (d *lvm) roundVolumeBlockSizeBytes(vol Volume, sizeBytes int64) int64 { +func (d *lvm) roundVolumeBlockSizeBytes(vol Volume, sizeBytes int64) (int64, error) { // Get the volume group's physical extent size, and use that as minimum size. - vgExtentSize, _ := d.volumeGroupExtentSize(d.config["lvm.vg_name"]) - return roundAbove(vgExtentSize, sizeBytes) + vgExtentSize, err := d.volumeGroupExtentSize(d.config["lvm.vg_name"]) + if err != nil { + return -1, err + } + + return roundAbove(vgExtentSize, sizeBytes), nil } diff --git a/internal/server/storage/drivers/driver_zfs.go b/internal/server/storage/drivers/driver_zfs.go index c435175c956..a600f2a2021 100644 --- a/internal/server/storage/drivers/driver_zfs.go +++ b/internal/server/storage/drivers/driver_zfs.go @@ -761,7 +761,7 @@ func (d *zfs) parseSource() (string, []string) { // roundVolumeBlockSizeBytes returns sizeBytes rounded up to the next multiple // of `vol`'s "zfs.blocksize". -func (d *zfs) roundVolumeBlockSizeBytes(vol Volume, sizeBytes int64) int64 { +func (d *zfs) roundVolumeBlockSizeBytes(vol Volume, sizeBytes int64) (int64, error) { minBlockSize, err := units.ParseByteSizeString(vol.ExpandedConfig("zfs.blocksize")) // minBlockSize will be 0 if zfs.blocksize="" @@ -770,5 +770,5 @@ func (d *zfs) roundVolumeBlockSizeBytes(vol Volume, sizeBytes int64) int64 { minBlockSize = 16 * 1024 } - return roundAbove(minBlockSize, sizeBytes) + return roundAbove(minBlockSize, sizeBytes), nil } diff --git a/internal/server/storage/drivers/driver_zfs_volumes.go b/internal/server/storage/drivers/driver_zfs_volumes.go index 0397d437739..d87f76ed348 100644 --- a/internal/server/storage/drivers/driver_zfs_volumes.go +++ b/internal/server/storage/drivers/driver_zfs_volumes.go @@ -88,7 +88,10 @@ func (d *zfs) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Oper } // Round to block boundary. - poolVolSizeBytes = d.roundVolumeBlockSizeBytes(vol, poolVolSizeBytes) + poolVolSizeBytes, err = d.roundVolumeBlockSizeBytes(vol, poolVolSizeBytes) + if err != nil { + return err + } // If the cached volume size is different than the pool volume size, then we can't use the // deleted cached image volume and instead we will rename it to a random UUID so it can't @@ -211,7 +214,10 @@ func (d *zfs) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Oper return err } - sizeBytes = d.roundVolumeBlockSizeBytes(vol, sizeBytes) + sizeBytes, err = d.roundVolumeBlockSizeBytes(vol, sizeBytes) + if err != nil { + return err + } // Create the volume dataset. err = d.createVolume(d.dataset(vol, false), sizeBytes, opts...) @@ -1689,7 +1695,10 @@ func (d *zfs) SetVolumeQuota(vol Volume, size string, allowUnsafeResize bool, op return nil } - sizeBytes = d.roundVolumeBlockSizeBytes(vol, sizeBytes) + sizeBytes, err = d.roundVolumeBlockSizeBytes(vol, sizeBytes) + if err != nil { + return err + } oldSizeBytesStr, err := d.getDatasetProperty(d.dataset(vol, false), "volsize") if err != nil { diff --git a/internal/server/storage/drivers/interface.go b/internal/server/storage/drivers/interface.go index f9ab979b244..cb6ff9fdecf 100644 --- a/internal/server/storage/drivers/interface.go +++ b/internal/server/storage/drivers/interface.go @@ -28,7 +28,7 @@ type Driver interface { // Internal. Info() Info HasVolume(vol Volume) (bool, error) - roundVolumeBlockSizeBytes(vol Volume, sizeBytes int64) int64 + roundVolumeBlockSizeBytes(vol Volume, sizeBytes int64) (int64, error) isBlockBacked(vol Volume) bool // Export struct details. diff --git a/internal/server/storage/drivers/utils.go b/internal/server/storage/drivers/utils.go index 04636642ac7..6c1a6b3e931 100644 --- a/internal/server/storage/drivers/utils.go +++ b/internal/server/storage/drivers/utils.go @@ -331,7 +331,11 @@ func ensureVolumeBlockFile(vol Volume, path string, sizeBytes int64, allowUnsafe } // Get rounded block size to avoid QEMU boundary issues. - sizeBytes = vol.driver.roundVolumeBlockSizeBytes(vol, sizeBytes) + var err error + sizeBytes, err = vol.driver.roundVolumeBlockSizeBytes(vol, sizeBytes) + if err != nil { + return false, err + } if util.PathExists(path) { fi, err := os.Stat(path) @@ -375,7 +379,7 @@ func ensureVolumeBlockFile(vol Volume, path string, sizeBytes int64, allowUnsafe // If path doesn't exist, then there has been no filler function supplied to create it from another source. // So instead create an empty volume (use for PXE booting a VM). - err := ensureSparseFile(path, sizeBytes) + err = ensureSparseFile(path, sizeBytes) if err != nil { return false, fmt.Errorf("Failed creating disk image %q as size %d: %w", path, sizeBytes, err) } diff --git a/internal/server/storage/drivers/volume.go b/internal/server/storage/drivers/volume.go index 22587158c21..f96ddbcd79d 100644 --- a/internal/server/storage/drivers/volume.go +++ b/internal/server/storage/drivers/volume.go @@ -530,7 +530,10 @@ func (v Volume) ConfigSizeFromSource(srcVol Volume) (string, error) { // directly usable with the same size setting without also rounding for this check. // Because we are not altering the actual size returned to use for the new volume, this will not // affect storage drivers that do not use rounding. - volSizeBytes = v.driver.roundVolumeBlockSizeBytes(v, volSizeBytes) + volSizeBytes, err = v.driver.roundVolumeBlockSizeBytes(v, volSizeBytes) + if err != nil { + return volSize, err + } // The volume/pool specified size is smaller than image minimum size. We must not continue as // these specified sizes provide protection against unpacking a massive image and filling the pool. From 51a07ac38ef1bf0625fd1b14907c92ac05da7f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Fri, 26 Jul 2024 18:18:29 -0400 Subject: [PATCH 03/16] incusd/instance_post: Fix cross-server live-migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- cmd/incusd/instance_post.go | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/cmd/incusd/instance_post.go b/cmd/incusd/instance_post.go index daaa07afa9a..ddd9613b81e 100644 --- a/cmd/incusd/instance_post.go +++ b/cmd/incusd/instance_post.go @@ -225,25 +225,27 @@ func instancePost(d *Daemon, r *http.Request) response.Response { } // Checks for running instances. - if inst.IsRunning() && (req.Pool != "" || req.Project != "" || target != "") { - // Stateless migrations need the instance stopped. - if !req.Live { - return response.BadRequest(fmt.Errorf("Instance must be stopped to be moved statelessly")) - } + if inst.IsRunning() { + if req.Pool != "" || req.Project != "" || target != "" { + // Stateless migrations need the instance stopped. + if !req.Live { + return response.BadRequest(fmt.Errorf("Instance must be stopped to be moved statelessly")) + } - // Storage pool changes require a stopped instance. - if req.Pool != "" { - return response.BadRequest(fmt.Errorf("Instance must be stopped to be moved across storage pools")) - } + // Storage pool changes require a stopped instance. + if req.Pool != "" { + return response.BadRequest(fmt.Errorf("Instance must be stopped to be moved across storage pools")) + } - // Project changes require a stopped instance. - if req.Project != "" { - return response.BadRequest(fmt.Errorf("Instance must be stopped to be moved across projects")) - } + // Project changes require a stopped instance. + if req.Project != "" { + return response.BadRequest(fmt.Errorf("Instance must be stopped to be moved across projects")) + } - // Name changes require a stopped instance. - if req.Name != "" { - return response.BadRequest(fmt.Errorf("Instance must be stopped to change their names")) + // Name changes require a stopped instance. + if req.Name != "" { + return response.BadRequest(fmt.Errorf("Instance must be stopped to change their names")) + } } } else { // Clear Live flag if instance isn't running. From 751ce3cc03055e78974847b662b653fca8f0872d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Fri, 26 Jul 2024 21:42:17 -0400 Subject: [PATCH 04/16] incus/image: Correct image copy logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The --project and --target-project logic was switched up. Signed-off-by: Stéphane Graber --- cmd/incus/image.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/cmd/incus/image.go b/cmd/incus/image.go index c885cb9ae51..b02b5174931 100644 --- a/cmd/incus/image.go +++ b/cmd/incus/image.go @@ -200,16 +200,6 @@ func (c *cmdImageCopy) Run(cmd *cobra.Command, args []string) error { return err } - // Revert project for `sourceServer` which may have been overwritten - // by `--project` flag in `GetImageServer` method - remote := conf.Remotes[remoteName] - if remote.Protocol == "incus" && !remote.Public { - d, ok := sourceServer.(incus.InstanceServer) - if ok { - sourceServer = d.UseProject(remote.Project) - } - } - // Parse destination remote resources, err := c.global.ParseServers(args[1]) if err != nil { @@ -228,8 +218,12 @@ func (c *cmdImageCopy) Run(cmd *cobra.Command, args []string) error { imageType = "virtual-machine" } + // Set the correct project on target. + remote := conf.Remotes[resources[0].remote] if c.flagTargetProject != "" { destinationServer = destinationServer.UseProject(c.flagTargetProject) + } else if remote.Protocol == "incus" { + destinationServer = destinationServer.UseProject(remote.Project) } // Copy the image From 98416d26ceca10044e9827bb7b7892f5c4ccb885 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Fri, 26 Jul 2024 22:18:39 -0400 Subject: [PATCH 05/16] incusd/storage/lvm: Hardden common functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/storage/drivers/driver_lvm_utils.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/server/storage/drivers/driver_lvm_utils.go b/internal/server/storage/drivers/driver_lvm_utils.go index 475e2988fd0..bcc476e34a9 100644 --- a/internal/server/storage/drivers/driver_lvm_utils.go +++ b/internal/server/storage/drivers/driver_lvm_utils.go @@ -133,7 +133,7 @@ func (d *lvm) volumeGroupExists(vgName string) (bool, []string, error) { // volumeGroupExtentSize gets the volume group's physical extent size in bytes. func (d *lvm) volumeGroupExtentSize(vgName string) (int64, error) { - output, err := subprocess.RunCommand("vgs", "--noheadings", "--nosuffix", "--units", "b", "-o", "vg_extent_size", vgName) + output, err := subprocess.TryRunCommand("vgs", "--noheadings", "--nosuffix", "--units", "b", "-o", "vg_extent_size", vgName) if err != nil { if d.isLVMNotFoundExitError(err) { return -1, api.StatusErrorf(http.StatusNotFound, "LVM volume group not found") @@ -148,7 +148,7 @@ func (d *lvm) volumeGroupExtentSize(vgName string) (int64, error) { // countLogicalVolumes gets the count of volumes (both normal and thin) in a volume group. func (d *lvm) countLogicalVolumes(vgName string) (int, error) { - output, err := subprocess.RunCommand("vgs", "--noheadings", "-o", "lv_count", vgName) + output, err := subprocess.TryRunCommand("vgs", "--noheadings", "-o", "lv_count", vgName) if err != nil { if d.isLVMNotFoundExitError(err) { return -1, api.StatusErrorf(http.StatusNotFound, "LVM volume group not found") @@ -163,7 +163,7 @@ func (d *lvm) countLogicalVolumes(vgName string) (int, error) { // countThinVolumes gets the count of thin volumes in a thin pool. func (d *lvm) countThinVolumes(vgName, poolName string) (int, error) { - output, err := subprocess.RunCommand("lvs", "--noheadings", "-o", "thin_count", fmt.Sprintf("%s/%s", vgName, poolName)) + output, err := subprocess.TryRunCommand("lvs", "--noheadings", "-o", "thin_count", fmt.Sprintf("%s/%s", vgName, poolName)) if err != nil { if d.isLVMNotFoundExitError(err) { return -1, api.StatusErrorf(http.StatusNotFound, "LVM volume group not found") From 36de828657f1d0c499354d825e78ae452ee53682 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Fri, 26 Jul 2024 23:23:23 -0400 Subject: [PATCH 06/16] incusd/api: Don't panic on missing config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- cmd/incusd/api.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/incusd/api.go b/cmd/incusd/api.go index 10d6ae1c0f8..917756fec46 100644 --- a/cmd/incusd/api.go +++ b/cmd/incusd/api.go @@ -383,6 +383,11 @@ func (s *httpServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } func setCORSHeaders(rw http.ResponseWriter, req *http.Request, config *clusterConfig.Config) { + // Check if we have a working config. + if config == nil { + return + } + allowedOrigin := config.HTTPSAllowedOrigin() origin := req.Header.Get("Origin") if allowedOrigin != "" && origin != "" { From b712fa05a3a9739863fb9ac3ce5c80e1b3a8f533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Fri, 26 Jul 2024 23:26:25 -0400 Subject: [PATCH 07/16] incusd/storage: Add Deactivate flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- cmd/incusd/storage_pools.go | 34 ++++++++++++++----- internal/server/storage/backend.go | 6 ++-- .../server/storage/drivers/driver_types.go | 1 + 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/cmd/incusd/storage_pools.go b/cmd/incusd/storage_pools.go index 885afeef9f6..06a76999223 100644 --- a/cmd/incusd/storage_pools.go +++ b/cmd/incusd/storage_pools.go @@ -1011,6 +1011,21 @@ func storagePoolDelete(d *Daemon, r *http.Request) response.Response { } } + // If the pool requires deactivation, go through it first. + if !clusterNotification && pool.Driver().Info().Remote && pool.Driver().Info().Deactivate { + err = notifier(func(client incus.InstanceServer) error { + _, _, err := client.GetServer() + if err != nil { + return err + } + + return client.DeleteStoragePool(pool.Name()) + }) + if err != nil { + return response.SmartError(err) + } + } + if pool.LocalStatus() != api.StoragePoolStatusPending { err = pool.Delete(clientType, nil) if err != nil { @@ -1024,15 +1039,18 @@ func storagePoolDelete(d *Daemon, r *http.Request) response.Response { return response.EmptySyncResponse } - // If we are clustered, also notify all other nodes. - err = notifier(func(client incus.InstanceServer) error { - _, _, err := client.GetServer() - if err != nil { - return err - } + // If clustered and dealing with a normal pool, notify all other nodes. + if !pool.Driver().Info().Remote || !pool.Driver().Info().Deactivate { + err = notifier(func(client incus.InstanceServer) error { + _, _, err := client.GetServer() + if err != nil { + return err + } + + return client.DeleteStoragePool(pool.Name()) + }) + } - return client.DeleteStoragePool(pool.Name()) - }) if err != nil { return response.SmartError(err) } diff --git a/internal/server/storage/backend.go b/internal/server/storage/backend.go index 09bcf0e93f6..54a1663c67d 100644 --- a/internal/server/storage/backend.go +++ b/internal/server/storage/backend.go @@ -353,12 +353,14 @@ func (b *backend) Delete(clientType request.ClientType, op *operations.Operation } if clientType != request.ClientTypeNormal && b.driver.Info().Remote { - if b.driver.Info().MountedRoot { + if b.driver.Info().Deactivate || b.driver.Info().MountedRoot { _, err := b.driver.Unmount() if err != nil { return err } - } else { + } + + if !b.driver.Info().MountedRoot { // Remote storage may have leftover entries caused by // volumes that were moved or delete while a particular system was offline. err := os.RemoveAll(path) diff --git a/internal/server/storage/drivers/driver_types.go b/internal/server/storage/drivers/driver_types.go index 4704b6a155b..c1d1a377fcc 100644 --- a/internal/server/storage/drivers/driver_types.go +++ b/internal/server/storage/drivers/driver_types.go @@ -18,6 +18,7 @@ type Info struct { DirectIO bool // Whether the driver supports direct I/O. IOUring bool // Whether the driver supports io_uring. MountedRoot bool // Whether the pool directory itself is a mount. + Deactivate bool // Whether an unmount action is required prior to removing the pool. } // VolumeFiller provides a struct for filling a volume. From 9b5b57c0e368182e2bc4caef556e6566ff00b688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Fri, 26 Jul 2024 23:26:45 -0400 Subject: [PATCH 08/16] incusd/storage/lvm: Add deactivation step for clusters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/storage/drivers/driver_lvm.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/internal/server/storage/drivers/driver_lvm.go b/internal/server/storage/drivers/driver_lvm.go index 3ebf3ea07f0..ab8c1db0134 100644 --- a/internal/server/storage/drivers/driver_lvm.go +++ b/internal/server/storage/drivers/driver_lvm.go @@ -128,6 +128,7 @@ func (d *lvm) Info() Info { IOUring: true, MountedRoot: false, Buckets: !d.isRemote(), + Deactivate: d.isRemote(), } } @@ -490,6 +491,11 @@ func (d *lvm) Delete(op *operations.Operation) error { // Remove volume group if needed. if removeVg { + if d.clustered { + // Wait for the locks we just released to clear. + time.Sleep(10 * time.Second) + } + _, err := subprocess.TryRunCommand("vgremove", "-f", d.config["lvm.vg_name"]) if err != nil { return fmt.Errorf("Failed to delete the volume group for the lvm storage pool: %w", err) @@ -684,7 +690,7 @@ func (d *lvm) Mount() (bool, error) { // If clustered LVM, start lock manager. if d.clustered { - _, err := subprocess.RunCommand("vgchange", "--lockstart") + _, err := subprocess.RunCommand("vgchange", "--lockstart", d.config["lvm.vg_name"]) if err != nil { return false, fmt.Errorf("Error starting lock manager: %w", err) } @@ -744,6 +750,13 @@ func (d *lvm) Mount() (bool, error) { // Unmount unmounts the storage pool (this does nothing). // LVM doesn't currently support unmounting, please see https://github.com/lxc/incus/issues/9278 func (d *lvm) Unmount() (bool, error) { + if d.clustered { + _, err := subprocess.RunCommand("vgchange", "--lockstop", d.config["lvm.vg_name"]) + if err != nil { + return false, fmt.Errorf("Error stopping lock manager: %w", err) + } + } + return false, nil } From 44d426939044a81f405db6dc5c15b75cc6398b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Sat, 27 Jul 2024 00:08:57 -0400 Subject: [PATCH 09/16] incusd/cluster: Return clear status for servers currently starting up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/cluster/connect.go | 2 +- internal/server/cluster/events.go | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/internal/server/cluster/connect.go b/internal/server/cluster/connect.go index e318f5753f2..ec79556aaa7 100644 --- a/internal/server/cluster/connect.go +++ b/internal/server/cluster/connect.go @@ -35,7 +35,7 @@ func Connect(address string, networkCert *localtls.CertInfo, serverCert *localtl defer cancel() err := EventListenerWait(ctx, address) if err != nil { - return nil, fmt.Errorf("Missing event connection with target cluster member") + return nil, err } } diff --git a/internal/server/cluster/events.go b/internal/server/cluster/events.go index 38406e340d7..0d20696aef6 100644 --- a/internal/server/cluster/events.go +++ b/internal/server/cluster/events.go @@ -2,6 +2,7 @@ package cluster import ( "context" + "fmt" "slices" "sync" "time" @@ -115,6 +116,7 @@ var eventHubAddresses []string var eventHubPushCh = make(chan api.Event, 10) // Buffer size to accommodate slow consumers before dropping events. var eventHubPushChTimeout = time.Duration(time.Second) var listeners = map[string]*eventListenerClient{} +var listenersUnavailable = map[string]bool{} var listenersNotify = map[chan struct{}][]string{} var listenersLock sync.Mutex var listenersUpdateLock sync.Mutex @@ -149,6 +151,11 @@ func EventListenerWait(ctx context.Context, address string) error { return nil } + if listenersUnavailable[address] { + listenersLock.Unlock() + return fmt.Errorf("Server isn't ready yet") + } + listenAddresses := []string{address} // Check if operating in event hub mode and if one of the event hub connections is available. @@ -181,7 +188,11 @@ func EventListenerWait(ctx context.Context, address string) error { case <-connected: return nil case <-ctx.Done(): - return ctx.Err() + if ctx.Err() != nil { + return fmt.Errorf("Missing event connection with target cluster member") + } + + return nil } } @@ -306,6 +317,9 @@ func EventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, l := logger.AddContext(logger.Ctx{"local": localAddress, "remote": m.Address}) if !HasConnectivity(endpoints.NetworkCert(), serverCert(), m.Address, true) { + listenersLock.Lock() + listenersUnavailable[m.Address] = true + listenersLock.Unlock() return } @@ -325,6 +339,7 @@ func EventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, listenersLock.Lock() listeners[m.Address] = listener + listenersUnavailable[m.Address] = false // Indicate to any notifiers waiting for this member's address that it is connected. for connected, notifyAddresses := range listenersNotify { From a80e104673ed3ecd746f1ee7fd1f74d0707695aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Sat, 27 Jul 2024 00:27:59 -0400 Subject: [PATCH 10/16] incusd/instance/lxc: Reduce logging level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/instance/drivers/driver_lxc.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/server/instance/drivers/driver_lxc.go b/internal/server/instance/drivers/driver_lxc.go index ce3b5edf7c8..79d7883b14c 100644 --- a/internal/server/instance/drivers/driver_lxc.go +++ b/internal/server/instance/drivers/driver_lxc.go @@ -3397,7 +3397,7 @@ func (d *lxc) Freeze() error { // Check if the CGroup is available if !d.state.OS.CGInfo.Supports(cgroup.Freezer, cg) { - d.logger.Info("Unable to freeze container (lack of kernel support)", ctxMap) + d.logger.Warn("Unable to freeze container (lack of kernel support)", ctxMap) return nil } @@ -3447,7 +3447,7 @@ func (d *lxc) Unfreeze() error { // Check if the CGroup is available if !d.state.OS.CGInfo.Supports(cgroup.Freezer, cg) { - d.logger.Info("Unable to unfreeze container (lack of kernel support)", ctxMap) + d.logger.Warn("Unable to unfreeze container (lack of kernel support)", ctxMap) return nil } @@ -5500,8 +5500,8 @@ fi } func (d *lxc) MigrateSend(args instance.MigrateSendArgs) error { - d.logger.Info("Migration send starting") - defer d.logger.Info("Migration send stopped") + d.logger.Debug("Migration send starting") + defer d.logger.Debug("Migration send stopped") // Wait for essential migration connections before negotiation. connectionsCtx, cancel := context.WithTimeout(context.Background(), time.Second*10) @@ -6087,8 +6087,8 @@ func (d *lxc) resetContainerDiskIdmap(srcIdmap *idmap.Set) error { } func (d *lxc) MigrateReceive(args instance.MigrateReceiveArgs) error { - d.logger.Info("Migration receive starting") - defer d.logger.Info("Migration receive stopped") + d.logger.Debug("Migration receive starting") + defer d.logger.Debug("Migration receive stopped") // Wait for essential migration connections before negotiation. connectionsCtx, cancel := context.WithTimeout(context.Background(), time.Second*10) @@ -6811,7 +6811,7 @@ func (d *lxc) migrate(args *instance.CriuMigrationArgs) error { if migrateErr != nil { log, err2 := getCRIULogErrors(finalStateDir, prettyCmd) if err2 == nil { - d.logger.Info("Failed migrating container", ctxMap) + d.logger.Warn("Failed migrating container", ctxMap) migrateErr = fmt.Errorf("%s %s failed\n%s", args.Function, prettyCmd, log) } From f533c369077ee8b49bd9c9ea5916cd60ba4cfd36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Sat, 27 Jul 2024 00:28:09 -0400 Subject: [PATCH 11/16] incusd/instance/qemu: Reduce logging level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/instance/drivers/driver_qemu.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/internal/server/instance/drivers/driver_qemu.go b/internal/server/instance/drivers/driver_qemu.go index 29c94e74e03..5a03cacbbbf 100644 --- a/internal/server/instance/drivers/driver_qemu.go +++ b/internal/server/instance/drivers/driver_qemu.go @@ -964,10 +964,7 @@ func (d *qemu) restoreState(monitor *qmp.Monitor) error { } go func() { - _, err := io.Copy(pipeWrite, stateConn) - if err != nil { - d.logger.Warn("Failed reading from state connection", logger.Ctx{"err": err}) - } + _, _ = io.Copy(pipeWrite, stateConn) _ = pipeRead.Close() _ = pipeWrite.Close() @@ -6426,8 +6423,8 @@ func (d *qemu) Export(w io.Writer, properties map[string]string, expiration time // MigrateSend is not currently supported. func (d *qemu) MigrateSend(args instance.MigrateSendArgs) error { - d.logger.Info("Migration send starting") - defer d.logger.Info("Migration send stopped") + d.logger.Debug("Migration send starting") + defer d.logger.Debug("Migration send stopped") // Check for stateful support. if args.Live && util.IsFalseOrEmpty(d.expandedConfig["migration.stateful"]) { @@ -7004,8 +7001,8 @@ func (d *qemu) migrateSendLive(pool storagePools.Pool, clusterMoveSourceName str } func (d *qemu) MigrateReceive(args instance.MigrateReceiveArgs) error { - d.logger.Info("Migration receive starting") - defer d.logger.Info("Migration receive stopped") + d.logger.Debug("Migration receive starting") + defer d.logger.Debug("Migration receive stopped") // Wait for essential migration connections before negotiation. connectionsCtx, cancel := context.WithTimeout(context.Background(), time.Second*10) From 3f02a65cc1cefb73acdda418dd3f37421cf64d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Sat, 27 Jul 2024 00:28:26 -0400 Subject: [PATCH 12/16] incusd/migrate: Reduce logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- cmd/incusd/migrate_instance.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/incusd/migrate_instance.go b/cmd/incusd/migrate_instance.go index e20eabc17d5..6479755e240 100644 --- a/cmd/incusd/migrate_instance.go +++ b/cmd/incusd/migrate_instance.go @@ -87,16 +87,16 @@ func (s *migrationSourceWs) Do(state *state.State, migrateOp *operations.Operati ctx, cancel := context.WithTimeout(context.TODO(), time.Second*10) defer cancel() - l.Info("Waiting for migration control connection on source") + l.Debug("Waiting for migration control connection on source") _, err := s.conns[api.SecretNameControl].WebSocket(ctx) if err != nil { return fmt.Errorf("Failed waiting for migration control connection on source: %w", err) } - l.Info("Migration control connection established on source") + l.Debug("Migration control connection established on source") - defer l.Info("Migration channels disconnected on source") + defer l.Debug("Migration channels disconnected on source") defer s.disconnect() stateConnFunc := func(ctx context.Context) (io.ReadWriteCloser, error) { @@ -215,16 +215,16 @@ func (c *migrationSink) Do(state *state.State, instOp *operationlock.InstanceOpe ctx, cancel := context.WithTimeout(context.TODO(), time.Second*10) defer cancel() - l.Info("Waiting for migration control connection on target") + l.Debug("Waiting for migration control connection on target") _, err := c.conns[api.SecretNameControl].WebSocket(ctx) if err != nil { return fmt.Errorf("Failed waiting for migration control connection on target: %w", err) } - l.Info("Migration control connection established on target") + l.Debug("Migration control connection established on target") - defer l.Info("Migration channels disconnected on target") + defer l.Debug("Migration channels disconnected on target") if c.push { defer c.disconnect() From bf993ca57aa389958239391aa9c511e9dfc394f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Sat, 27 Jul 2024 00:28:33 -0400 Subject: [PATCH 13/16] incusd/storage: Reduce logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/storage/backend.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/server/storage/backend.go b/internal/server/storage/backend.go index 54a1663c67d..5bf2c807d57 100644 --- a/internal/server/storage/backend.go +++ b/internal/server/storage/backend.go @@ -4913,7 +4913,7 @@ func (b *backend) migrationIndexHeaderSend(l logger.Logger, indexHeaderVersion u return nil, fmt.Errorf("Failed negotiating migration options: %w", err) } - l.Info("Received migration index header response", logger.Ctx{"response": fmt.Sprintf("%+v", infoResp), "version": indexHeaderVersion}) + l.Debug("Received migration index header response", logger.Ctx{"response": fmt.Sprintf("%+v", infoResp), "version": indexHeaderVersion}) } return &infoResp, nil @@ -4938,7 +4938,7 @@ func (b *backend) migrationIndexHeaderReceive(l logger.Logger, indexHeaderVersio return nil, fmt.Errorf("Failed decoding migration index header: %w", err) } - l.Info("Received migration index header, sending response", logger.Ctx{"version": indexHeaderVersion}) + l.Debug("Received migration index header, sending response", logger.Ctx{"version": indexHeaderVersion}) infoResp := localMigration.InfoResponse{StatusCode: http.StatusOK, Refresh: &refresh} headerJSON, err := json.Marshal(infoResp) From ef22ae3cd5fd7a291ae6a2075f565031eab3a3ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Sat, 27 Jul 2024 00:55:44 -0400 Subject: [PATCH 14/16] incusd/instance/qemu: Remove double lifecycle event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/instance/drivers/driver_qemu.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/server/instance/drivers/driver_qemu.go b/internal/server/instance/drivers/driver_qemu.go index 5a03cacbbbf..08319b58ea5 100644 --- a/internal/server/instance/drivers/driver_qemu.go +++ b/internal/server/instance/drivers/driver_qemu.go @@ -4779,8 +4779,6 @@ func (d *qemu) Stop(stateful bool) error { return err } - d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceStopped.Event(d, nil)) - op.Done(nil) return nil } From 8366dfbe8026f75d4c23cb757b90b12f686614fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Sat, 27 Jul 2024 01:22:05 -0400 Subject: [PATCH 15/16] tests/clustering: Use correct target project argument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- test/suites/clustering.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/suites/clustering.sh b/test/suites/clustering.sh index 6defedd5da8..1c4836902c8 100644 --- a/test/suites/clustering.sh +++ b/test/suites/clustering.sh @@ -2747,7 +2747,7 @@ test_clustering_image_refresh() { for project in default foo bar; do # Copy the public image to each project - INCUS_DIR="${INCUS_ONE_DIR}" incus image copy public:testimage local: --alias testimage --project "${project}" + INCUS_DIR="${INCUS_ONE_DIR}" incus image copy public:testimage local: --alias testimage --target-project "${project}" # Diable autoupdate for testimage in project foo if [ "${project}" = "foo" ]; then From 8e33c788fa48ffceaf5a4f91eeb9835141dd7862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Sat, 27 Jul 2024 01:36:07 -0400 Subject: [PATCH 16/16] incusd/isntance/edk2: Fix CSM handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/instance/drivers/edk2/driver_edk2.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/server/instance/drivers/edk2/driver_edk2.go b/internal/server/instance/drivers/edk2/driver_edk2.go index 292a41d7e48..094f1584c13 100644 --- a/internal/server/instance/drivers/edk2/driver_edk2.go +++ b/internal/server/instance/drivers/edk2/driver_edk2.go @@ -57,6 +57,7 @@ var architectureInstallations = map[int][]Installation{ {Code: "OVMF_CODE.fd", Vars: "qemu.nvram"}, }, CSM: { + {Code: "seabios.bin", Vars: "seabios.bin"}, {Code: "OVMF_CODE.4MB.CSM.fd", Vars: "OVMF_VARS.4MB.CSM.fd"}, {Code: "OVMF_CODE.csm.4m.fd", Vars: "OVMF_VARS.4m.fd"}, {Code: "OVMF_CODE.2MB.CSM.fd", Vars: "OVMF_VARS.2MB.CSM.fd"}, @@ -68,7 +69,6 @@ var architectureInstallations = map[int][]Installation{ Path: "/usr/share/qemu", Usage: map[FirmwareUsage][]FirmwarePair{ GENERIC: { - {Code: "seabios.bin", Vars: "seabios.bin"}, {Code: "ovmf-x86_64-4m-code.bin", Vars: "ovmf-x86_64-4m-vars.bin"}, {Code: "ovmf-x86_64.bin", Vars: "ovmf-x86_64-code.bin"}, }, @@ -76,6 +76,9 @@ var architectureInstallations = map[int][]Installation{ {Code: "ovmf-x86_64-ms-4m-vars.bin", Vars: "ovmf-x86_64-ms-4m-code.bin"}, {Code: "ovmf-x86_64-ms-code.bin", Vars: "ovmf-x86_64-ms-vars.bin"}, }, + CSM: { + {Code: "seabios.bin", Vars: "seabios.bin"}, + }, }, }, { Path: "/usr/share/OVMF/x64",