diff --git a/api/src/main/java/com/cloud/capacity/Capacity.java b/api/src/main/java/com/cloud/capacity/Capacity.java index a4e2c2a7f05d..4e584b18feee 100644 --- a/api/src/main/java/com/cloud/capacity/Capacity.java +++ b/api/src/main/java/com/cloud/capacity/Capacity.java @@ -34,13 +34,17 @@ public interface Capacity extends InternalIdentity, Identity { public static final short CAPACITY_TYPE_LOCAL_STORAGE = 9; public static final short CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET = 10; public static final short CAPACITY_TYPE_GPU = 19; + public static final short CAPACITY_TYPE_OBJECT_STORAGE = 20; + public static final short CAPACITY_TYPE_BACKUP_STORAGE = 21; public static final short CAPACITY_TYPE_CPU_CORE = 90; public static final List STORAGE_CAPACITY_TYPES = List.of(CAPACITY_TYPE_STORAGE, CAPACITY_TYPE_STORAGE_ALLOCATED, CAPACITY_TYPE_SECONDARY_STORAGE, - CAPACITY_TYPE_LOCAL_STORAGE); + CAPACITY_TYPE_LOCAL_STORAGE, + CAPACITY_TYPE_BACKUP_STORAGE, + CAPACITY_TYPE_OBJECT_STORAGE); public Long getHostOrPoolId(); diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index a8777d5c75b8..be21f13267b1 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -632,11 +632,13 @@ public class EventTypes { public static final String EVENT_VM_BACKUP_CREATE = "BACKUP.CREATE"; public static final String EVENT_VM_BACKUP_RESTORE = "BACKUP.RESTORE"; public static final String EVENT_VM_BACKUP_DELETE = "BACKUP.DELETE"; + public static final String EVENT_VM_BACKUP_OFFERING_REMOVED_AND_BACKUPS_DELETED = "BACKUP.OFFERING.BACKUPS.DEL"; public static final String EVENT_VM_BACKUP_RESTORE_VOLUME_TO_VM = "BACKUP.RESTORE.VOLUME.TO.VM"; public static final String EVENT_VM_BACKUP_SCHEDULE_CONFIGURE = "BACKUP.SCHEDULE.CONFIGURE"; public static final String EVENT_VM_BACKUP_SCHEDULE_DELETE = "BACKUP.SCHEDULE.DELETE"; public static final String EVENT_VM_BACKUP_USAGE_METRIC = "BACKUP.USAGE.METRIC"; public static final String EVENT_VM_BACKUP_EDIT = "BACKUP.OFFERING.EDIT"; + public static final String EVENT_VM_CREATE_FROM_BACKUP = "VM.CREATE.FROM.BACKUP"; // external network device events public static final String EVENT_EXTERNAL_NVP_CONTROLLER_ADD = "PHYSICAL.NVPCONTROLLER.ADD"; diff --git a/api/src/main/java/com/cloud/network/NetworkService.java b/api/src/main/java/com/cloud/network/NetworkService.java index fd51cbfa774b..196e1f9aab86 100644 --- a/api/src/main/java/com/cloud/network/NetworkService.java +++ b/api/src/main/java/com/cloud/network/NetworkService.java @@ -272,5 +272,7 @@ Network createPrivateNetwork(String networkName, String displayText, long physic boolean handleCksIsoOnNetworkVirtualRouter(Long virtualRouterId, boolean mount) throws ResourceUnavailableException; + IpAddresses getIpAddressesFromIps(String ipAddress, String ip6Address, String macAddress); + String getNicVlanValueForExternalVm(NicTO nic); } diff --git a/api/src/main/java/com/cloud/offering/DiskOfferingInfo.java b/api/src/main/java/com/cloud/offering/DiskOfferingInfo.java index d83039e15c2b..12dcf423e34f 100644 --- a/api/src/main/java/com/cloud/offering/DiskOfferingInfo.java +++ b/api/src/main/java/com/cloud/offering/DiskOfferingInfo.java @@ -31,6 +31,13 @@ public DiskOfferingInfo(DiskOffering diskOffering) { _diskOffering = diskOffering; } + public DiskOfferingInfo(DiskOffering diskOffering, Long size, Long minIops, Long maxIops) { + _diskOffering = diskOffering; + _size = size; + _minIops = minIops; + _maxIops = maxIops; + } + public void setDiskOffering(DiskOffering diskOffering) { _diskOffering = diskOffering; } diff --git a/api/src/main/java/com/cloud/storage/StorageService.java b/api/src/main/java/com/cloud/storage/StorageService.java index 6f7b62911b62..a29c8f6aecef 100644 --- a/api/src/main/java/com/cloud/storage/StorageService.java +++ b/api/src/main/java/com/cloud/storage/StorageService.java @@ -134,7 +134,7 @@ public interface StorageService { void removeSecondaryStorageHeuristic(RemoveSecondaryStorageSelectorCmd cmd); - ObjectStore discoverObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException; + ObjectStore discoverObjectStore(String name, String url, Long size, String providerName, Map details) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException; boolean deleteObjectStore(DeleteObjectStoragePoolCmd cmd); diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index a8ed62fb6b96..6f1aba4613da 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd; import org.apache.cloudstack.api.command.admin.vm.RecoverVMCmd; import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; +import org.apache.cloudstack.api.command.user.vm.CreateVMFromBackupCmd; import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; import org.apache.cloudstack.api.command.user.vm.RebootVMCmd; @@ -220,7 +221,7 @@ void startVirtualMachineForHA(VirtualMachine vm, Map securityGroupIdList, - Account owner, String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor, HTTPMethod httpmethod, + Account owner, String hostName, String displayName, Long diskOfferingId, Long diskSize, List dataDiskInfoList, String group, HypervisorType hypervisor, HTTPMethod httpmethod, String userData, Long userDataId, String userDataDetails, List sshKeyPairs, Map requestedIps, IpAddresses defaultIp, Boolean displayVm, String keyboard, List affinityGroupIdList, Map customParameter, String customId, Map> dhcpOptionMap, Map dataDiskTemplateToDiskOfferingMap, @@ -297,7 +298,7 @@ UserVm createBasicSecurityGroupVirtualMachine(DataCenter zone, ServiceOffering s * available. */ UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, ServiceOffering serviceOffering, VirtualMachineTemplate template, List networkIdList, - List securityGroupIdList, Account owner, String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor, + List securityGroupIdList, Account owner, String hostName, String displayName, Long diskOfferingId, Long diskSize, List dataDiskInfoList, String group, HypervisorType hypervisor, HTTPMethod httpmethod, String userData, Long userDataId, String userDataDetails, List sshKeyPairs, Map requestedIps, IpAddresses defaultIps, Boolean displayVm, String keyboard, List affinityGroupIdList, Map customParameters, String customId, Map> dhcpOptionMap, Map dataDiskTemplateToDiskOfferingMap, Map userVmOVFProperties, boolean dynamicScalingEnabled, Long overrideDiskOfferingId, String vmType, Volume volume, Snapshot snapshot) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException, StorageUnavailableException, ResourceAllocationException; @@ -369,7 +370,7 @@ UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, ServiceOfferin * available. */ UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serviceOffering, VirtualMachineTemplate template, List networkIdList, Account owner, - String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor, HTTPMethod httpmethod, String userData, + String hostName, String displayName, Long diskOfferingId, Long diskSize, List dataDiskInfoList, String group, HypervisorType hypervisor, HTTPMethod httpmethod, String userData, Long userDataId, String userDataDetails, List sshKeyPairs, Map requestedIps, IpAddresses defaultIps, Boolean displayVm, String keyboard, List affinityGroupIdList, Map customParameters, String customId, Map> dhcpOptionMap, Map dataDiskTemplateToDiskOfferingMap, Map templateOvfPropertiesMap, boolean dynamicScalingEnabled, String vmType, Long overrideDiskOfferingId, Volume volume, Snapshot snapshot) @@ -516,4 +517,8 @@ UserVm importVM(final DataCenter zone, final Host host, final VirtualMachineTemp * @return true if the VM is successfully unmanaged, false if not. */ boolean unmanageUserVM(Long vmId); + + UserVm allocateVMFromBackup(CreateVMFromBackupCmd cmd) throws InsufficientCapacityException, ResourceAllocationException, ResourceUnavailableException; + + UserVm restoreVMFromBackup(CreateVMFromBackupCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException, ResourceAllocationException; } diff --git a/api/src/main/java/com/cloud/vm/VirtualMachine.java b/api/src/main/java/com/cloud/vm/VirtualMachine.java index e2ea408e7b8c..d244de7115e8 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachine.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachine.java @@ -128,7 +128,6 @@ public static StateMachine2 getStat s_fsm.addTransition(new Transition(State.Error, VirtualMachine.Event.DestroyRequested, State.Expunging, null)); s_fsm.addTransition(new Transition(State.Error, VirtualMachine.Event.ExpungeOperation, State.Expunging, null)); s_fsm.addTransition(new Transition(State.Stopped, Event.RestoringRequested, State.Restoring, null)); - s_fsm.addTransition(new Transition(State.Expunging, Event.RestoringRequested, State.Restoring, null)); s_fsm.addTransition(new Transition(State.Destroyed, Event.RestoringRequested, State.Restoring, null)); s_fsm.addTransition(new Transition(State.Restoring, Event.RestoringSuccess, State.Stopped, null)); s_fsm.addTransition(new Transition(State.Restoring, Event.RestoringFailed, State.Stopped, null)); diff --git a/api/src/main/java/com/cloud/vm/VmDiskInfo.java b/api/src/main/java/com/cloud/vm/VmDiskInfo.java new file mode 100644 index 000000000000..b8779a8d77c6 --- /dev/null +++ b/api/src/main/java/com/cloud/vm/VmDiskInfo.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.vm; + +import com.cloud.offering.DiskOffering; +import com.cloud.offering.DiskOfferingInfo; + +public class VmDiskInfo extends DiskOfferingInfo { + private Long _deviceId; + + public VmDiskInfo(DiskOffering diskOffering, Long size, Long minIops, Long maxIops) { + super(diskOffering, size, minIops, maxIops); + } + + public VmDiskInfo(DiskOffering diskOffering, Long size, Long minIops, Long maxIops, Long deviceId) { + super(diskOffering, size, minIops, maxIops); + _deviceId = deviceId; + } + + public Long getDeviceId() { + return _deviceId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/alert/AlertService.java b/api/src/main/java/org/apache/cloudstack/alert/AlertService.java index 5146e5c38e8e..d8e471756a02 100644 --- a/api/src/main/java/org/apache/cloudstack/alert/AlertService.java +++ b/api/src/main/java/org/apache/cloudstack/alert/AlertService.java @@ -74,6 +74,8 @@ private AlertType(short type, String name, boolean isDefault) { public static final AlertType ALERT_TYPE_VR_PUBLIC_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PUBLIC.IFACE.MTU", true); public static final AlertType ALERT_TYPE_VR_PRIVATE_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PRIVATE.IFACE.MTU", true); public static final AlertType ALERT_TYPE_EXTENSION_PATH_NOT_READY = new AlertType((short)33, "ALERT.TYPE.EXTENSION.PATH.NOT.READY", true); + public static final AlertType ALERT_TYPE_BACKUP_STORAGE = new AlertType(Capacity.CAPACITY_TYPE_BACKUP_STORAGE, "ALERT.STORAGE.BACKUP", true); + public static final AlertType ALERT_TYPE_OBJECT_STORAGE = new AlertType(Capacity.CAPACITY_TYPE_OBJECT_STORAGE, "ALERT.STORAGE.OBJECT", true); public short getType() { return type; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index a8e45c8112e6..3b36ce9100e3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -29,6 +29,7 @@ public class ApiConstants { public static final String ADDRESS = "address"; public static final String ALGORITHM = "algorithm"; public static final String ALIAS = "alias"; + public static final String ALLOCATED = "allocated"; public static final String ALLOCATED_DATE = "allocateddate"; public static final String ALLOCATED_ONLY = "allocatedonly"; public static final String ALLOCATED_TIME = "allocated"; @@ -60,6 +61,7 @@ public class ApiConstants { public static final String BACKUP_STORAGE_AVAILABLE = "backupstorageavailable"; public static final String BACKUP_STORAGE_LIMIT = "backupstoragelimit"; public static final String BACKUP_STORAGE_TOTAL = "backupstoragetotal"; + public static final String BACKUP_VM_OFFERING_REMOVED = "vmbackupofferingremoved"; public static final String BACKUP_TOTAL = "backuptotal"; public static final String BASE64_IMAGE = "base64image"; public static final String BGP_PEERS = "bgppeers"; @@ -155,6 +157,7 @@ public class ApiConstants { public static final String MAX_IOPS = "maxiops"; public static final String HYPERVISOR_SNAPSHOT_RESERVE = "hypervisorsnapshotreserve"; public static final String DATACENTER_NAME = "datacentername"; + public static final String DATADISKS_DETAILS = "datadisksdetails"; public static final String DATADISK_OFFERING_LIST = "datadiskofferinglist"; public static final String DEFAULT_VALUE = "defaultvalue"; public static final String DELETE_PROTECTION = "deleteprotection"; @@ -163,6 +166,7 @@ public class ApiConstants { public static final String DESTINATION_ZONE_ID = "destzoneid"; public static final String DETAILS = "details"; public static final String DEVICE_ID = "deviceid"; + public static final String DEVICE_IDS = "deviceids"; public static final String DEVICE_NAME = "devicename"; public static final String DIRECT_DOWNLOAD = "directdownload"; public static final String DISK = "disk"; @@ -305,6 +309,7 @@ public class ApiConstants { public static final String IP_ADDRESS = "ipaddress"; public static final String IP_ADDRESSES = "ipaddresses"; public static final String IP6_ADDRESS = "ip6address"; + public static final String IP6_ADDRESSES = "ip6addresses"; public static final String IP_ADDRESS_ID = "ipaddressid"; public static final String IS_2FA_ENABLED = "is2faenabled"; public static final String IS_2FA_VERIFIED = "is2faverified"; @@ -356,6 +361,7 @@ public class ApiConstants { public static final String LBID = "lbruleid"; public static final String LB_PROVIDER = "lbprovider"; public static final String MAC_ADDRESS = "macaddress"; + public static final String MAC_ADDRESSES = "macaddresses"; public static final String MANUAL_UPGRADE = "manualupgrade"; public static final String MAX = "max"; public static final String MAX_SNAPS = "maxsnaps"; @@ -383,6 +389,7 @@ public class ApiConstants { public static final String NETMASK = "netmask"; public static final String NEW_NAME = "newname"; public static final String NIC = "nic"; + public static final String NICS = "nics"; public static final String NIC_NETWORK_LIST = "nicnetworklist"; public static final String NIC_IP_ADDRESS_LIST = "nicipaddresslist"; public static final String NIC_MULTIQUEUE_NUMBER = "nicmultiqueuenumber"; @@ -459,6 +466,7 @@ public class ApiConstants { public static final String POWER_STATE = "powerstate"; public static final String PRECEDENCE = "precedence"; public static final String PREPARE_VM = "preparevm"; + public static final String PRESERVE_IP = "preserveip"; public static final String PRIVATE_INTERFACE = "privateinterface"; public static final String PRIVATE_IP = "privateip"; public static final String PRIVATE_PORT = "privateport"; @@ -494,6 +502,7 @@ public class ApiConstants { public static final String REGISTERED = "registered"; public static final String QUALIFIERS = "qualifiers"; public static final String QUERY_FILTER = "queryfilter"; + public static final String QUIESCE_VM = "quiescevm"; public static final String SCHEDULE = "schedule"; public static final String SCHEDULE_ID = "scheduleid"; public static final String SCOPE = "scope"; @@ -509,6 +518,7 @@ public class ApiConstants { public static final String SERIAL = "serial"; public static final String SERVICE_IP = "serviceip"; public static final String SERVICE_OFFERING_ID = "serviceofferingid"; + public static final String SERVICE_OFFERING_NAME = "serviceofferingname"; public static final String SESSIONKEY = "sessionkey"; public static final String SHOW_CAPACITIES = "showcapacities"; public static final String SHOW_REMOVED = "showremoved"; @@ -576,6 +586,7 @@ public class ApiConstants { public static final String TRUST_STORE_PASSWORD = "truststorepass"; public static final String URL = "url"; public static final String USAGE_INTERFACE = "usageinterface"; + public static final String USED = "used"; public static final String USED_SUBNETS = "usedsubnets"; public static final String USED_IOPS = "usediops"; public static final String USER_DATA = "userdata"; @@ -634,8 +645,10 @@ public class ApiConstants { public static final String IS_DEDICATED = "isdedicated"; public static final String TAKEN = "taken"; public static final String VM_AVAILABLE = "vmavailable"; + public static final String VM_DETAILS = "vmdetails"; public static final String VM_LIMIT = "vmlimit"; public static final String VM_TOTAL = "vmtotal"; + public static final String VM_SETTINGS = "vmsettings"; public static final String VM_TYPE = "vmtype"; public static final String VNET = "vnet"; public static final String IS_VOLATILE = "isvolatile"; @@ -817,6 +830,7 @@ public class ApiConstants { public static final String LIST_ALL = "listall"; public static final String LIST_ONLY_REMOVED = "listonlyremoved"; public static final String LIST_SYSTEM_VMS = "listsystemvms"; + public static final String LIST_VM_DETAILS = "listvmdetails"; public static final String IP_RANGES = "ipranges"; public static final String IPV4_ROUTING = "ip4routing"; public static final String IPV4_ROUTES = "ip4routes"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java index d0683299e73f..51918570078e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java @@ -38,7 +38,6 @@ import org.apache.cloudstack.api.response.AutoScaleVmProfileResponse; import org.apache.cloudstack.api.response.BackupOfferingResponse; import org.apache.cloudstack.api.response.BackupRepositoryResponse; -import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.BackupScheduleResponse; import org.apache.cloudstack.api.response.BucketResponse; import org.apache.cloudstack.api.response.CapacityResponse; @@ -142,7 +141,6 @@ import org.apache.cloudstack.api.response.VpcResponse; import org.apache.cloudstack.api.response.VpnUsersResponse; import org.apache.cloudstack.api.response.ZoneResponse; -import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupOffering; import org.apache.cloudstack.backup.BackupRepository; import org.apache.cloudstack.backup.BackupSchedule; @@ -534,8 +532,6 @@ List createTemplateResponses(ResponseView view, VirtualMachine UserDataResponse createUserDataResponse(UserData userData); - BackupResponse createBackupResponse(Backup backup); - BackupScheduleResponse createBackupScheduleResponse(BackupSchedule backup); BackupOfferingResponse createBackupOfferingResponse(BackupOffering policy); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java index e65326cd874d..f46cecdef504 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java @@ -141,7 +141,6 @@ public int compare(CapacityResponse resp1, CapacityResponse resp2) { } }); - response.setResponses(capacityResponses); response.setResponseName(getCommandName()); this.setResponseObject(response); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java index b779ba2a2b47..460b8d642e90 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java @@ -56,6 +56,9 @@ public class AddObjectStoragePoolCmd extends BaseCmd { @Parameter(name = ApiConstants.TAGS, type = CommandType.STRING, description = "the tags for the storage pool") private String tags; + @Parameter(name = ApiConstants.SIZE, type = CommandType.LONG, description = "the total size of the object store in GiB. Used for tracking capacity and sending alerts", since = "4.21") + private Long size; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -68,6 +71,10 @@ public String getName() { return name; } + public Long getTotalSize() { + return size; + } + public Map getDetails() { Map detailsMap = null; if (details != null && !details.isEmpty()) { @@ -112,7 +119,7 @@ public long getEntityOwnerId() { @Override public void execute(){ try{ - ObjectStore result = _storageService.discoverObjectStore(getName(), getUrl(), getProviderName(), getDetails()); + ObjectStore result = _storageService.discoverObjectStore(getName(), getUrl(), getTotalSize(), getProviderName(), getDetails()); ObjectStoreResponse storeResponse = null; if (result != null) { storeResponse = _responseGenerator.createObjectStoreResponse(result); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java index 497179d25ef1..ac007137ef1f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java @@ -44,6 +44,8 @@ public class UpdateObjectStoragePoolCmd extends BaseCmd { @Parameter(name = ApiConstants.URL, type = CommandType.STRING, description = "the url for the object store") private String url; + @Parameter(name = ApiConstants.SIZE, type = CommandType.LONG, description = "the total size of the object store in GiB. Used for tracking capacity and sending alerts. Set to 0 to stop tracking.", since = "4.21") + private Long size; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -61,6 +63,10 @@ public String getUrl() { return url; } + public Long getSize() { + return size; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/CreateVMFromBackupCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/CreateVMFromBackupCmdByAdmin.java new file mode 100644 index 000000000000..d95f17ef304c --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/CreateVMFromBackupCmdByAdmin.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.vm; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.command.user.vm.CreateVMFromBackupCmd; +import org.apache.cloudstack.api.response.ClusterResponse; +import org.apache.cloudstack.api.response.PodResponse; +import org.apache.cloudstack.api.response.UserVmResponse; + +import com.cloud.vm.VirtualMachine; + +@APICommand(name = "createVMFromBackup", + description = "Creates and automatically starts a VM from a backup.", + responseObject = UserVmResponse.class, + responseView = ResponseObject.ResponseView.Full, + entityType = {VirtualMachine.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, + since = "4.21.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class CreateVMFromBackupCmdByAdmin extends CreateVMFromBackupCmd implements AdminCmd { + + @Parameter(name = ApiConstants.POD_ID, type = CommandType.UUID, entityType = PodResponse.class, description = "destination Pod ID to deploy the VM to - parameter available for root admin only", since = "4.21") + private Long podId; + + @Parameter(name = ApiConstants.CLUSTER_ID, type = CommandType.UUID, entityType = ClusterResponse.class, description = "destination Cluster ID to deploy the VM to - parameter available for root admin only", since = "4.21") + private Long clusterId; + + public Long getPodId() { + return podId; + } + + public Long getClusterId() { + return clusterId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java index af25f873f87a..4e9028f52173 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java @@ -60,6 +60,27 @@ public class CreateBackupCmd extends BaseAsyncCreateCmd { description = "ID of the VM") private Long vmId; + @Parameter(name = ApiConstants.NAME, + type = CommandType.STRING, + description = "the name of the backup", + since = "4.21.0") + private String name; + + @Parameter(name = ApiConstants.DESCRIPTION, + type = CommandType.STRING, + description = "the description for the backup", + since = "4.21.0") + private String description; + + @Parameter(name = ApiConstants.QUIESCE_VM, + type = CommandType.BOOLEAN, + required = false, + description = "Quiesce the instance before checkpointing the disks for backup. Applicable only to NAS backup provider. " + + "The filesystem is frozen before the backup starts and thawed immediately after. " + + "Requires the instance to have the QEMU Guest Agent installed and running.", + since = "4.21.0") + private Boolean quiesceVM; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -68,6 +89,18 @@ public Long getVmId() { return vmId; } + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public Boolean getQuiesceVM() { + return quiesceVM; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -75,7 +108,7 @@ public Long getVmId() { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { try { - boolean result = backupManager.createBackup(getVmId(), getJob()); + boolean result = backupManager.createBackup(this, getJob()); if (result) { SuccessResponse response = new SuccessResponse(getCommandName()); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java index f9903eaf1eca..a352c688f127 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java @@ -79,6 +79,15 @@ public class CreateBackupScheduleCmd extends BaseCmd { since = "4.21.0", description = ApiConstants.PARAMETER_DESCRIPTION_MAX_BACKUPS) private Integer maxBackups; + @Parameter(name = ApiConstants.QUIESCE_VM, + type = CommandType.BOOLEAN, + required = false, + description = "Quiesce the instance before checkpointing the disks for backup. Applicable only to NAS backup provider. " + + "The filesystem is frozen before the backup starts and thawed immediately after. " + + "Requires the instance to have the QEMU Guest Agent installed and running.", + since = "4.21.0") + private Boolean quiesceVM; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -103,6 +112,10 @@ public Integer getMaxBackups() { return maxBackups; } + public Boolean getQuiesceVM() { + return quiesceVM; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupsCmd.java index 7d87cc37e6cd..0c2b4f9b91ef 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupsCmd.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupOfferingResponse; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.UserVmResponse; @@ -75,6 +76,25 @@ public class ListBackupsCmd extends BaseListProjectAndAccountResourcesCmd { description = "list backups by zone id") private Long zoneId; + @Parameter(name = ApiConstants.NAME, + type = CommandType.STRING, + since = "4.21.0", + description = "list backups by name") + private String name; + + @Parameter(name = ApiConstants.BACKUP_OFFERING_ID, + type = CommandType.UUID, + entityType = BackupOfferingResponse.class, + since = "4.21.0", + description = "list backups by backup offering") + private Long backupOfferingId; + + @Parameter(name = ApiConstants.LIST_VM_DETAILS, + type = CommandType.BOOLEAN, + since = "4.21.0", + description = "list backups with VM details") + private Boolean listVmDetails; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -87,10 +107,22 @@ public Long getVmId() { return vmId; } + public String getName() { + return name; + } + + public Long getBackupOfferingId() { + return backupOfferingId; + } + public Long getZoneId() { return zoneId; } + public Boolean getListVmDetails() { + return listVmDetails; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -101,7 +133,7 @@ protected void setupResponseBackupList(final List backups, final Integer if (backup == null) { continue; } - BackupResponse backupResponse = _responseGenerator.createBackupResponse(backup); + BackupResponse backupResponse = backupManager.createBackupResponse(backup, this.getListVmDetails()); responses.add(backupResponse); } final ListResponse response = new ListResponse<>(); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java new file mode 100644 index 000000000000..ecbde47692f2 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java @@ -0,0 +1,848 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.vm; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nonnull; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.affinity.AffinityGroupResponse; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.ApiArgValidator; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; +import org.apache.cloudstack.api.BaseAsyncCreateCustomIdCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.DiskOfferingResponse; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.HostResponse; +import org.apache.cloudstack.api.response.NetworkResponse; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.api.response.SecurityGroupResponse; +import org.apache.cloudstack.api.response.UserDataResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.vm.lease.VMLeaseManager; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.EnumUtils; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.agent.api.LogLevel; +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.network.Network; +import com.cloud.network.Network.IpAddresses; +import com.cloud.offering.DiskOffering; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.net.NetUtils; +import com.cloud.vm.VmDetailConstants; +import com.cloud.vm.VmDiskInfo; +import com.cloud.utils.net.Dhcp; + +public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityGroupAction, UserCmd { + + private static final String s_name = "deployvirtualmachineresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, required = true, description = "availability zone for the virtual machine") + private Long zoneId; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "host name for the virtual machine", validations = {ApiArgValidator.RFCComplianceDomainName}) + private String name; + + @Parameter(name = ApiConstants.DISPLAY_NAME, type = CommandType.STRING, description = "an optional user generated name for the virtual machine") + private String displayName; + + @Parameter(name=ApiConstants.PASSWORD, type=CommandType.STRING, description="The password of the virtual machine. If null, a random password will be generated for the VM.", + since="4.19.0.0") + protected String password; + + //Owner information + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the virtual machine. Must be used with domainId.") + private String accountName; + + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used. If account is NOT provided then virtual machine will be assigned to the caller account and domain.") + private Long domainId; + + //Network information + //@ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.NETWORK_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = NetworkResponse.class, description = "list of network ids used by virtual machine. Can't be specified with ipToNetworkList parameter") + private List networkIds; + + @Parameter(name = ApiConstants.BOOT_TYPE, type = CommandType.STRING, required = false, description = "Guest VM Boot option either custom[UEFI] or default boot [BIOS]. Not applicable with VMware if the template is marked as deploy-as-is, as we honour what is defined in the template.", since = "4.14.0.0") + private String bootType; + + @Parameter(name = ApiConstants.BOOT_MODE, type = CommandType.STRING, required = false, description = "Boot Mode [Legacy] or [Secure] Applicable when Boot Type Selected is UEFI, otherwise Legacy only for BIOS. Not applicable with VMware if the template is marked as deploy-as-is, as we honour what is defined in the template.", since = "4.14.0.0") + private String bootMode; + + @Parameter(name = ApiConstants.BOOT_INTO_SETUP, type = CommandType.BOOLEAN, required = false, description = "Boot into hardware setup or not (ignored if startVm = false, only valid for vmware)", since = "4.15.0.0") + private Boolean bootIntoSetup; + + //DataDisk information + @ACL + @Parameter(name = ApiConstants.DISK_OFFERING_ID, type = CommandType.UUID, entityType = DiskOfferingResponse.class, description = "the ID of the disk offering for the virtual machine. If the template is of ISO format," + + " the diskOfferingId is for the root disk volume. Otherwise this parameter is used to indicate the " + + "offering for the data disk volume. If the templateId parameter passed is from a Template object," + + " the diskOfferingId refers to a DATA Disk Volume created. If the templateId parameter passed is " + + "from an ISO object, the diskOfferingId refers to a ROOT Disk Volume created.") + private Long diskOfferingId; + + @Parameter(name = ApiConstants.SIZE, type = CommandType.LONG, description = "the arbitrary size for the DATADISK volume. Mutually exclusive with diskOfferingId") + private Long size; + + @Parameter(name = ApiConstants.ROOT_DISK_SIZE, + type = CommandType.LONG, + description = "Optional field to resize root disk on deploy. Value is in GB. Only applies to template-based deployments. Analogous to details[0].rootdisksize, which takes precedence over this parameter if both are provided", + since = "4.4") + private Long rootdisksize; + + @Parameter(name = ApiConstants.DATADISKS_DETAILS, + type = CommandType.MAP, + since = "4.21.0", + description = "Disk offering details for creating multiple data volumes. Mutually exclusive with diskOfferingId." + + " Example: datadisksdetails[0].diskofferingid=a2a73a84-19db-4852-8930-dfddef053341&datadisksdetails[0].size=10&datadisksdetails[0].miniops=100&datadisksdetails[0].maxiops=200") + private Map dataDisksDetails; + + @Parameter(name = ApiConstants.GROUP, type = CommandType.STRING, description = "an optional group for the virtual machine") + private String group; + + @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, description = "the hypervisor on which to deploy the virtual machine. " + + "The parameter is required and respected only when hypervisor info is not set on the ISO/Template passed to the call") + private String hypervisor; + + @Parameter(name = ApiConstants.USER_DATA, type = CommandType.STRING, + description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " + + "This binary data must be base64 encoded before adding it to the request. " + + "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " + + "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " + + "You also need to change vm.userdata.max.length value", + length = 1048576) + private String userData; + + @Parameter(name = ApiConstants.USER_DATA_ID, type = CommandType.UUID, entityType = UserDataResponse.class, description = "the ID of the Userdata", since = "4.18") + private Long userdataId; + + @Parameter(name = ApiConstants.USER_DATA_DETAILS, type = CommandType.MAP, description = "used to specify the parameters values for the variables in userdata.", since = "4.18") + private Map userdataDetails; + + @Deprecated + @Parameter(name = ApiConstants.SSH_KEYPAIR, type = CommandType.STRING, description = "name of the ssh key pair used to login to the virtual machine") + private String sshKeyPairName; + + @Parameter(name = ApiConstants.SSH_KEYPAIRS, type = CommandType.LIST, collectionType = CommandType.STRING, since="4.17", description = "names of the ssh key pairs used to login to the virtual machine") + private List sshKeyPairNames; + + @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, description = "destination Host ID to deploy the VM to - parameter available for root admin only") + private Long hostId; + + @ACL + @Parameter(name = ApiConstants.SECURITY_GROUP_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = SecurityGroupResponse.class, description = "comma separated list of security groups id that going to be applied to the virtual machine. " + + "Should be passed only when vm is created from a zone with Basic Network support." + " Mutually exclusive with securitygroupnames parameter") + private List securityGroupIdList; + + @ACL + @Parameter(name = ApiConstants.SECURITY_GROUP_NAMES, type = CommandType.LIST, collectionType = CommandType.STRING, entityType = SecurityGroupResponse.class, description = "comma separated list of security groups names that going to be applied to the virtual machine." + + " Should be passed only when vm is created from a zone with Basic Network support. " + "Mutually exclusive with securitygroupids parameter") + private List securityGroupNameList; + + @Parameter(name = ApiConstants.IP_NETWORK_LIST, type = CommandType.MAP, description = "ip to network mapping. Can't be specified with networkIds parameter." + + " Example: iptonetworklist[0].ip=10.10.10.11&iptonetworklist[0].ipv6=fc00:1234:5678::abcd&iptonetworklist[0].networkid=uuid&iptonetworklist[0].mac=aa:bb:cc:dd:ee::ff - requests to use ip 10.10.10.11 in network id=uuid") + private Map ipToNetworkList; + + @Parameter(name = ApiConstants.IP_ADDRESS, type = CommandType.STRING, description = "the ip address for default vm's network") + private String ipAddress; + + @Parameter(name = ApiConstants.IP6_ADDRESS, type = CommandType.STRING, description = "the ipv6 address for default vm's network") + private String ip6Address; + + @Parameter(name = ApiConstants.MAC_ADDRESS, type = CommandType.STRING, description = "the mac address for default vm's network") + private String macAddress; + + @Parameter(name = ApiConstants.KEYBOARD, type = CommandType.STRING, description = "an optional keyboard device type for the virtual machine. valid value can be one of de,de-ch,es,fi,fr,fr-be,fr-ch,is,it,jp,nl-be,no,pt,uk,us") + private String keyboard; + + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "Deploy vm for the project") + private Long projectId; + + @Parameter(name = ApiConstants.START_VM, type = CommandType.BOOLEAN, description = "true if start vm after creating; defaulted to true if not specified") + private Boolean startVm; + + @ACL + @Parameter(name = ApiConstants.AFFINITY_GROUP_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = AffinityGroupResponse.class, description = "comma separated list of affinity groups id that are going to be applied to the virtual machine." + + " Mutually exclusive with affinitygroupnames parameter") + private List affinityGroupIdList; + + @ACL + @Parameter(name = ApiConstants.AFFINITY_GROUP_NAMES, type = CommandType.LIST, collectionType = CommandType.STRING, entityType = AffinityGroupResponse.class, description = "comma separated list of affinity groups names that are going to be applied to the virtual machine." + + "Mutually exclusive with affinitygroupids parameter") + private List affinityGroupNameList; + + @Parameter(name = ApiConstants.DISPLAY_VM, type = CommandType.BOOLEAN, since = "4.2", description = "an optional field, whether to the display the vm to the end user or not.", authorized = {RoleType.Admin}) + private Boolean displayVm; + + @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, since = "4.3", description = "used to specify the custom parameters. 'extraconfig' is not allowed to be passed in details") + private Map details; + + @Parameter(name = ApiConstants.DEPLOYMENT_PLANNER, type = CommandType.STRING, description = "Deployment planner to use for vm allocation. Available to ROOT admin only", since = "4.4", authorized = { RoleType.Admin }) + private String deploymentPlanner; + + @Parameter(name = ApiConstants.DHCP_OPTIONS_NETWORK_LIST, type = CommandType.MAP, description = "DHCP options which are passed to the VM on start up" + + " Example: dhcpoptionsnetworklist[0].dhcp:114=url&dhcpoptionsetworklist[0].networkid=networkid&dhcpoptionsetworklist[0].dhcp:66=www.test.com") + private Map dhcpOptionsNetworkList; + + @Parameter(name = ApiConstants.DATADISK_OFFERING_LIST, type = CommandType.MAP, since = "4.11", description = "datadisk template to disk-offering mapping;" + + " an optional parameter used to create additional data disks from datadisk templates; can't be specified with diskOfferingId parameter") + private Map dataDiskTemplateToDiskOfferingList; + + @Parameter(name = ApiConstants.EXTRA_CONFIG, type = CommandType.STRING, since = "4.12", description = "an optional URL encoded string that can be passed to the virtual machine upon successful deployment", length = 5120) + private String extraConfig; + + @Parameter(name = ApiConstants.COPY_IMAGE_TAGS, type = CommandType.BOOLEAN, since = "4.13", description = "if true the image tags (if any) will be copied to the VM, default value is false") + private Boolean copyImageTags; + + @Parameter(name = ApiConstants.PROPERTIES, type = CommandType.MAP, since = "4.15", + description = "used to specify the vApp properties.") + @LogLevel(LogLevel.Log4jLevel.Off) + private Map vAppProperties; + + @Parameter(name = ApiConstants.NIC_NETWORK_LIST, type = CommandType.MAP, since = "4.15", + description = "VMware only: used to specify network mapping of a vApp VMware template registered \"as-is\"." + + " Example nicnetworklist[0].ip=Nic-101&nicnetworklist[0].network=uuid") + @LogLevel(LogLevel.Log4jLevel.Off) + private Map vAppNetworks; + + @Parameter(name = ApiConstants.DYNAMIC_SCALING_ENABLED, type = CommandType.BOOLEAN, since = "4.16", + description = "true if virtual machine needs to be dynamically scalable") + protected Boolean dynamicScalingEnabled; + + @Parameter(name = ApiConstants.OVERRIDE_DISK_OFFERING_ID, type = CommandType.UUID, since = "4.17", entityType = DiskOfferingResponse.class, description = "the ID of the disk offering for the virtual machine to be used for root volume instead of the disk offering mapped in service offering." + + "In case of virtual machine deploying from ISO, then the diskofferingid specified for root volume is ignored and uses this override disk offering id") + private Long overrideDiskOfferingId; + + @Parameter(name = ApiConstants.IOTHREADS_ENABLED, type = CommandType.BOOLEAN, required = false, + description = "IOThreads are dedicated event loop threads for supported disk devices to perform block I/O requests in order to improve scalability especially on an SMP host/guest with many LUNs.") + private Boolean iothreadsEnabled; + + @Parameter(name = ApiConstants.IO_DRIVER_POLICY, type = CommandType.STRING, description = "Controls specific policies on IO") + private String ioDriverPolicy; + + @Parameter(name = ApiConstants.NIC_MULTIQUEUE_NUMBER, type = CommandType.INTEGER, since = "4.18", + description = "The number of queues for multiqueue NICs.") + private Integer nicMultiqueueNumber; + + @Parameter(name = ApiConstants.NIC_PACKED_VIRTQUEUES_ENABLED, type = CommandType.BOOLEAN, since = "4.18", + description = "Enable packed virtqueues or not.") + private Boolean nicPackedVirtQueues; + + @Parameter(name = ApiConstants.INSTANCE_LEASE_DURATION, type = CommandType.INTEGER, since = "4.21.0", + description = "Number of days instance is leased for.") + private Integer leaseDuration; + + @Parameter(name = ApiConstants.INSTANCE_LEASE_EXPIRY_ACTION, type = CommandType.STRING, since = "4.21.0", + description = "Lease expiry action, valid values are STOP and DESTROY") + private String leaseExpiryAction; + + @Parameter(name = ApiConstants.EXTERNAL_DETAILS, + type = CommandType.MAP, + description = "Details in key/value pairs using format externaldetails[i].keyname=keyvalue. Example: externaldetails[0].server.type=typevalue", + since = "4.21.0") + protected Map externalDetails; + + private List dataDiskInfoList; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return s_name; + } + + public String getAccountName() { + if (accountName == null) { + return CallContext.current().getCallingAccount().getAccountName(); + } + return accountName; + } + + public Long getDiskOfferingId() { + return diskOfferingId; + } + + public String getDeploymentPlanner() { + return deploymentPlanner; + } + + public String getDisplayName() { + return displayName; + } + + public Long getDomainId() { + if (domainId == null) { + return CallContext.current().getCallingAccount().getDomainId(); + } + return domainId; + } + + public ApiConstants.BootType getBootType() { + if (StringUtils.isNotBlank(bootType)) { + try { + String type = bootType.trim().toUpperCase(); + return ApiConstants.BootType.valueOf(type); + } catch (IllegalArgumentException e) { + String errMesg = "Invalid bootType " + bootType + "Specified for vm " + getName() + + " Valid values are: " + Arrays.toString(ApiConstants.BootType.values()); + logger.warn(errMesg); + throw new InvalidParameterValueException(errMesg); + } + } + return null; + } + + public Map getDetails() { + Map customparameterMap = convertDetailsToMap(details); + + if (getBootType() != null) { + customparameterMap.put(getBootType().toString(), getBootMode().toString()); + } + + if (rootdisksize != null && !customparameterMap.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) { + customparameterMap.put(VmDetailConstants.ROOT_DISK_SIZE, rootdisksize.toString()); + } + + IoDriverPolicy ioPolicy = getIoDriverPolicy(); + if (ioPolicy != null) { + customparameterMap.put(VmDetailConstants.IO_POLICY, ioPolicy.toString()); + } + + if (BooleanUtils.toBoolean(iothreadsEnabled)) { + customparameterMap.put(VmDetailConstants.IOTHREADS, BooleanUtils.toStringTrueFalse(iothreadsEnabled)); + } + + if (nicMultiqueueNumber != null) { + customparameterMap.put(VmDetailConstants.NIC_MULTIQUEUE_NUMBER, nicMultiqueueNumber.toString()); + } + + if (BooleanUtils.toBoolean(nicPackedVirtQueues)) { + customparameterMap.put(VmDetailConstants.NIC_PACKED_VIRTQUEUES_ENABLED, BooleanUtils.toStringTrueFalse(nicPackedVirtQueues)); + } + + if (MapUtils.isNotEmpty(externalDetails)) { + customparameterMap.putAll(getExternalDetails()); + } + return customparameterMap; + } + + public Map getExternalDetails() { + return convertExternalDetailsToMap(externalDetails); + } + + public ApiConstants.BootMode getBootMode() { + if (StringUtils.isNotBlank(bootMode)) { + try { + String mode = bootMode.trim().toUpperCase(); + return ApiConstants.BootMode.valueOf(mode); + } catch (IllegalArgumentException e) { + String msg = String.format("Invalid %s: %s specified for VM: %s. Valid values are: %s", + ApiConstants.BOOT_MODE, bootMode, getName(), Arrays.toString(ApiConstants.BootMode.values())); + logger.error(msg); + throw new InvalidParameterValueException(msg); + } + } + if (ApiConstants.BootType.UEFI.equals(getBootType())) { + String msg = String.format("%s must be specified for the VM with boot type: %s. Valid values are: %s", + ApiConstants.BOOT_MODE, getBootType(), Arrays.toString(ApiConstants.BootMode.values())); + logger.error(msg); + throw new InvalidParameterValueException(msg); + } + return null; + } + + public Map getVmProperties() { + Map map = new HashMap<>(); + if (MapUtils.isNotEmpty(vAppProperties)) { + Collection parameterCollection = vAppProperties.values(); + Iterator iterator = parameterCollection.iterator(); + while (iterator.hasNext()) { + HashMap entry = (HashMap)iterator.next(); + map.put(entry.get("key"), entry.get("value")); + } + } + return map; + } + + public Map getVmNetworkMap() { + Map map = new HashMap<>(); + if (MapUtils.isNotEmpty(vAppNetworks)) { + Collection parameterCollection = vAppNetworks.values(); + Iterator iterator = parameterCollection.iterator(); + while (iterator.hasNext()) { + HashMap entry = (HashMap) iterator.next(); + Integer nic; + try { + nic = Integer.valueOf(entry.get(VmDetailConstants.NIC)); + } catch (NumberFormatException nfe) { + nic = null; + } + String networkUuid = entry.get(VmDetailConstants.NETWORK); + if (logger.isTraceEnabled()) { + logger.trace(String.format("nic, '%s', goes on net, '%s'", nic, networkUuid)); + } + if (nic == null || StringUtils.isEmpty(networkUuid) || _entityMgr.findByUuid(Network.class, networkUuid) == null) { + throw new InvalidParameterValueException(String.format("Network ID: %s for NIC ID: %s is invalid", networkUuid, nic)); + } + map.put(nic, _entityMgr.findByUuid(Network.class, networkUuid).getId()); + } + } + return map; + } + + public String getGroup() { + return group; + } + + public HypervisorType getHypervisor() { + return HypervisorType.getType(hypervisor); + } + + public Boolean isDisplayVm() { + return displayVm; + } + + @Override + public boolean isDisplay() { + if(displayVm == null) + return true; + else + return displayVm; + } + + public List getSecurityGroupNameList() { + return securityGroupNameList; + } + + public List getSecurityGroupIdList() { + return securityGroupIdList; + } + + public Long getSize() { + return size; + } + + public String getUserData() { + return userData; + } + + public Long getUserdataId() { + return userdataId; + } + + public Map getUserdataDetails() { + return convertDetailsToMap(userdataDetails); + } + + public Long getZoneId() { + return zoneId; + } + + public String getPassword() { + return password; + } + + public Integer getLeaseDuration() { + return leaseDuration; + } + + public VMLeaseManager.ExpiryAction getLeaseExpiryAction() { + if (StringUtils.isBlank(leaseExpiryAction)) { + return null; + } + VMLeaseManager.ExpiryAction action = EnumUtils.getEnumIgnoreCase(VMLeaseManager.ExpiryAction.class, leaseExpiryAction); + if (action == null) { + throw new InvalidParameterValueException("Invalid value configured for leaseexpiryaction, valid values are: " + + com.cloud.utils.EnumUtils.listValues(VMLeaseManager.ExpiryAction.values())); + } + return action; + } + + public List getNetworkIds() { + if (MapUtils.isNotEmpty(vAppNetworks)) { + if (CollectionUtils.isNotEmpty(networkIds) || ipAddress != null || getIp6Address() != null || MapUtils.isNotEmpty(ipToNetworkList)) { + throw new InvalidParameterValueException(String.format("%s can't be specified along with %s, %s, %s", ApiConstants.NIC_NETWORK_LIST, ApiConstants.NETWORK_IDS, ApiConstants.IP_ADDRESS, ApiConstants.IP_NETWORK_LIST)); + } else { + return new ArrayList<>(); + } + } + if (ipToNetworkList != null && !ipToNetworkList.isEmpty()) { + if ((networkIds != null && !networkIds.isEmpty()) || ipAddress != null || getIp6Address() != null) { + throw new InvalidParameterValueException("ipToNetworkMap can't be specified along with networkIds or ipAddress"); + } else { + List networks = new ArrayList(); + networks.addAll(getIpToNetworkMap().keySet()); + return networks; + } + } + return networkIds; + } + + public String getName() { + return name; + } + + public List getSSHKeyPairNames() { + List sshKeyPairs = new ArrayList(); + if(sshKeyPairNames != null) { + sshKeyPairs = sshKeyPairNames; + } + if(sshKeyPairName != null && !sshKeyPairName.isEmpty()) { + sshKeyPairs.add(sshKeyPairName); + } + return sshKeyPairs; + } + + public List getDataDiskInfoList() { + if (this.dataDiskInfoList != null) { + return this.dataDiskInfoList; + } + if (dataDisksDetails == null || dataDisksDetails.isEmpty()) { + return null; + } + if (dataDiskTemplateToDiskOfferingList != null) { + throw new InvalidParameterValueException("datadisktemplatetodiskofferinglist parameter can't be specified along with datadisksdetails parameter"); + } + List vmDiskInfoList = new ArrayList<>(); + Collection dataDisksCollection = dataDisksDetails.values(); + Iterator iter = dataDisksCollection.iterator(); + while (iter.hasNext()) { + HashMap dataDisk = (HashMap)iter.next(); + String diskOfferingUuid = dataDisk.get(ApiConstants.DISK_OFFERING_ID); + if (diskOfferingUuid == null) { + throw new InvalidParameterValueException("diskofferingid parameter is required for datadiskdetails"); + } + DiskOffering diskOffering = _entityMgr.findByUuid(DiskOffering.class, diskOfferingUuid); + if (diskOffering == null) { + throw new InvalidParameterValueException("Unable to find disk offering " + diskOfferingUuid); + } + if (diskOffering.isComputeOnly()) { + throw new InvalidParameterValueException(String.format("The disk offering id %d provided is directly mapped to a service offering, please provide an individual disk offering", diskOffering.getUuid())); + } + + Long size = null; + Long minIops = null; + Long maxIops = null; + if (dataDisk.get(ApiConstants.DEVICE_ID) == null) { + throw new InvalidParameterValueException("deviceid parameter is required for datadiskdetails"); + } + Long deviceId = Long.parseLong(dataDisk.get(ApiConstants.DEVICE_ID)); + if (diskOffering.isCustomized()) { + if (dataDisk.get(ApiConstants.SIZE) == null) { + throw new InvalidParameterValueException("Size is required for custom disk offering"); + } + size = Long.parseLong(dataDisk.get(ApiConstants.SIZE)); + } else { + size = diskOffering.getDiskSize() / (1024 * 1024 * 1024); + } + if (diskOffering.isCustomizedIops() != null && diskOffering.isCustomizedIops()) { + if (dataDisk.get(ApiConstants.MIN_IOPS) == null) { + throw new InvalidParameterValueException("Min IOPS is required for custom disk offering"); + } + if (dataDisk.get(ApiConstants.MAX_IOPS) == null) { + throw new InvalidParameterValueException("Max IOPS is required for custom disk offering"); + } + minIops = Long.parseLong(dataDisk.get(ApiConstants.MIN_IOPS)); + maxIops = Long.parseLong(dataDisk.get(ApiConstants.MAX_IOPS)); + } + VmDiskInfo vmDiskInfo = new VmDiskInfo(diskOffering, size, minIops, maxIops, deviceId); + vmDiskInfoList.add(vmDiskInfo); + } + this.dataDiskInfoList = vmDiskInfoList; + return dataDiskInfoList; + } + + public Long getHostId() { + return hostId; + } + + public boolean getStartVm() { + return startVm == null ? true : startVm; + } + + public Map getIpToNetworkMap() { + if ((networkIds != null || ipAddress != null || getIp6Address() != null) && ipToNetworkList != null) { + throw new InvalidParameterValueException("NetworkIds and ipAddress can't be specified along with ipToNetworkMap parameter"); + } + LinkedHashMap ipToNetworkMap = null; + if (ipToNetworkList != null && !ipToNetworkList.isEmpty()) { + ipToNetworkMap = new LinkedHashMap(); + Collection ipsCollection = ipToNetworkList.values(); + Iterator iter = ipsCollection.iterator(); + while (iter.hasNext()) { + HashMap ips = (HashMap)iter.next(); + Long networkId = getNetworkIdFomIpMap(ips); + IpAddresses addrs = getIpAddressesFromIpMap(ips); + ipToNetworkMap.put(networkId, addrs); + } + } + + return ipToNetworkMap; + } + + @Nonnull + private IpAddresses getIpAddressesFromIpMap(HashMap ips) { + String requestedIp = ips.get("ip"); + String requestedIpv6 = ips.get("ipv6"); + String requestedMac = ips.get("mac"); + if (requestedIpv6 != null) { + requestedIpv6 = NetUtils.standardizeIp6Address(requestedIpv6); + } + if (requestedMac != null) { + if(!NetUtils.isValidMac(requestedMac)) { + throw new InvalidParameterValueException("Mac address is not valid: " + requestedMac); + } else if(!NetUtils.isUnicastMac(requestedMac)) { + throw new InvalidParameterValueException("Mac address is not unicast: " + requestedMac); + } + requestedMac = NetUtils.standardizeMacAddress(requestedMac); + } + return new IpAddresses(requestedIp, requestedIpv6, requestedMac); + } + + @Nonnull + private Long getNetworkIdFomIpMap(HashMap ips) { + Long networkId; + final String networkid = ips.get("networkid"); + Network network = _networkService.getNetwork(networkid); + if (network != null) { + networkId = network.getId(); + } else { + try { + networkId = Long.parseLong(networkid); + } catch (NumberFormatException e) { + throw new InvalidParameterValueException("Unable to translate and find entity with networkId: " + networkid); + } + } + return networkId; + } + + public String getIpAddress() { + return ipAddress; + } + + public String getIp6Address() { + if (ip6Address == null) { + return null; + } + return NetUtils.standardizeIp6Address(ip6Address); + } + + + public String getMacAddress() { + if (macAddress == null) { + return null; + } + if(!NetUtils.isValidMac(macAddress)) { + throw new InvalidParameterValueException("Mac address is not valid: " + macAddress); + } else if(!NetUtils.isUnicastMac(macAddress)) { + throw new InvalidParameterValueException("Mac address is not unicast: " + macAddress); + } + return NetUtils.standardizeMacAddress(macAddress); + } + + public List getAffinityGroupIdList() { + if (affinityGroupNameList != null && affinityGroupIdList != null) { + throw new InvalidParameterValueException("affinitygroupids parameter is mutually exclusive with affinitygroupnames parameter"); + } + + // transform group names to ids here + if (affinityGroupNameList != null) { + List affinityGroupIds = new ArrayList(); + for (String groupName : affinityGroupNameList) { + Long groupId = _responseGenerator.getAffinityGroupId(groupName, getEntityOwnerId()); + if (groupId == null) { + throw new InvalidParameterValueException("Unable to find affinity group by name " + groupName); + } else { + affinityGroupIds.add(groupId); + } + } + return affinityGroupIds; + } else { + return affinityGroupIdList; + } + } + + public String getKeyboard() { + // TODO Auto-generated method stub + return keyboard; + } + + public Map> getDhcpOptionsMap() { + Map> dhcpOptionsMap = new HashMap<>(); + if (dhcpOptionsNetworkList != null && !dhcpOptionsNetworkList.isEmpty()) { + + Collection> paramsCollection = this.dhcpOptionsNetworkList.values(); + for (Map dhcpNetworkOptions : paramsCollection) { + String networkId = dhcpNetworkOptions.get(ApiConstants.NETWORK_ID); + + if (networkId == null) { + throw new IllegalArgumentException("No networkid specified when providing extra dhcp options."); + } + + Map dhcpOptionsForNetwork = new HashMap<>(); + dhcpOptionsMap.put(networkId, dhcpOptionsForNetwork); + + for (String key : dhcpNetworkOptions.keySet()) { + if (key.startsWith(ApiConstants.DHCP_PREFIX)) { + int dhcpOptionValue = Integer.parseInt(key.replaceFirst(ApiConstants.DHCP_PREFIX, "")); + dhcpOptionsForNetwork.put(dhcpOptionValue, dhcpNetworkOptions.get(key)); + } else if (!key.equals(ApiConstants.NETWORK_ID)) { + Dhcp.DhcpOptionCode dhcpOptionEnum = Dhcp.DhcpOptionCode.valueOfString(key); + dhcpOptionsForNetwork.put(dhcpOptionEnum.getCode(), dhcpNetworkOptions.get(key)); + } + } + + } + } + + return dhcpOptionsMap; + } + + public Map getDataDiskTemplateToDiskOfferingMap() { + if (diskOfferingId != null && dataDiskTemplateToDiskOfferingList != null) { + throw new InvalidParameterValueException("diskofferingid parameter can't be specified along with datadisktemplatetodiskofferinglist parameter"); + } + if (MapUtils.isEmpty(dataDiskTemplateToDiskOfferingList)) { + return new HashMap(); + } + + HashMap dataDiskTemplateToDiskOfferingMap = new HashMap(); + for (Object objDataDiskTemplates : dataDiskTemplateToDiskOfferingList.values()) { + HashMap dataDiskTemplates = (HashMap) objDataDiskTemplates; + Long dataDiskTemplateId; + DiskOffering dataDiskOffering = null; + VirtualMachineTemplate dataDiskTemplate= _entityMgr.findByUuid(VirtualMachineTemplate.class, dataDiskTemplates.get("datadisktemplateid")); + if (dataDiskTemplate == null) { + dataDiskTemplate = _entityMgr.findById(VirtualMachineTemplate.class, dataDiskTemplates.get("datadisktemplateid")); + if (dataDiskTemplate == null) + throw new InvalidParameterValueException("Unable to translate and find entity with datadisktemplateid " + dataDiskTemplates.get("datadisktemplateid")); + } + dataDiskTemplateId = dataDiskTemplate.getId(); + dataDiskOffering = _entityMgr.findByUuid(DiskOffering.class, dataDiskTemplates.get("diskofferingid")); + if (dataDiskOffering == null) { + dataDiskOffering = _entityMgr.findById(DiskOffering.class, dataDiskTemplates.get("diskofferingid")); + if (dataDiskOffering == null) + throw new InvalidParameterValueException("Unable to translate and find entity with diskofferingId " + dataDiskTemplates.get("diskofferingid")); + } + dataDiskTemplateToDiskOfferingMap.put(dataDiskTemplateId, dataDiskOffering); + } + return dataDiskTemplateToDiskOfferingMap; + } + + public String getExtraConfig() { + return extraConfig; + } + + public boolean getCopyImageTags() { + return copyImageTags == null ? false : copyImageTags; + } + + public Boolean getBootIntoSetup() { + return bootIntoSetup; + } + + public boolean isDynamicScalingEnabled() { + return dynamicScalingEnabled == null ? true : dynamicScalingEnabled; + } + + public Long getOverrideDiskOfferingId() { + return overrideDiskOfferingId; + } + + public IoDriverPolicy getIoDriverPolicy() { + if (StringUtils.isNotBlank(ioDriverPolicy)) { + try { + String policyType = ioDriverPolicy.trim().toUpperCase(); + return IoDriverPolicy.valueOf(policyType); + } catch (IllegalArgumentException e) { + String errMesg = String.format("Invalid io policy %s specified for vm %s. Valid values are: %s", ioDriverPolicy, getName(), Arrays.toString(IoDriverPolicy.values())); + logger.warn(errMesg); + throw new InvalidParameterValueException(errMesg); + } + } + return null; + } + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + public static String getResultObjectName() { + return "virtualmachine"; + } + + @Override + public long getEntityOwnerId() { + Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + if (accountId == null) { + return CallContext.current().getCallingAccount().getId(); + } + + return accountId; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_CREATE; + } + + @Override + public String getCreateEventType() { + return EventTypes.EVENT_VM_CREATE; + } + + @Override + public String getCreateEventDescription() { + return "creating Vm"; + } + + @Override + public String getEventDescription() { + if(getStartVm()) { + return "starting Vm. Vm Id: " + getEntityUuid(); + } + return "deploying Vm. Vm Id: " + getEntityUuid(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java new file mode 100644 index 000000000000..04e413ed67ab --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.vm; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.cloudstack.api.response.UserVmResponse; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InsufficientServerCapacityException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.uservm.UserVm; +import com.cloud.vm.VirtualMachine; + +@APICommand(name = "createVMFromBackup", + description = "Creates and automatically starts a VM from a backup.", + responseObject = UserVmResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {VirtualMachine.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, + since = "4.21.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class CreateVMFromBackupCmd extends BaseDeployVMCmd { + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.BACKUP_ID, + type = CommandType.UUID, + entityType = BackupResponse.class, + required = true, + description = "backup ID to create the VM from") + private Long backupId; + + @ACL + @Parameter(name = ApiConstants.SERVICE_OFFERING_ID, type = CommandType.UUID, entityType = ServiceOfferingResponse.class, description = "the ID of the service offering for the virtual machine") + private Long serviceOfferingId; + + @ACL + @Parameter(name = ApiConstants.TEMPLATE_ID, type = CommandType.UUID, entityType = TemplateResponse.class, description = "the ID of the template for the virtual machine") + private Long templateId; + + @Parameter(name = ApiConstants.PRESERVE_IP, type = CommandType.BOOLEAN, description = "Use the same IP/MAC addresses as stored in the backup metadata. Works only if the original Instance is deleted and the IP/MAC address is available.") + private Boolean preserveIp; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getBackupId() { + return backupId; + } + + public Long getServiceOfferingId() { + return serviceOfferingId; + } + + public Long getTemplateId() { + return templateId; + } + + public boolean getPreserveIp() { + return (preserveIp != null) ? preserveIp : false; + } + + @Override + public void create() { + UserVm vm; + try { + vm = _userVmService.allocateVMFromBackup(this); + if (vm != null) { + setEntityId(vm.getId()); + setEntityUuid(vm.getUuid()); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to deploy vm"); + } + } catch (InsufficientCapacityException ex) { + logger.info(ex); + logger.trace(ex.getMessage(), ex); + throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, ex.getMessage()); + } catch (ResourceUnavailableException ex) { + logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, ex.getMessage()); + } catch (ConcurrentOperationException ex) { + logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } catch (ResourceAllocationException ex) { + logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.RESOURCE_ALLOCATION_ERROR, ex.getMessage()); + } + } + + @Override + public void execute () { + UserVm vm = null; + try { + vm = _userVmService.restoreVMFromBackup(this); + } catch (ResourceUnavailableException ex) { + logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, ex.getMessage()); + } catch (ResourceAllocationException ex) { + logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.RESOURCE_ALLOCATION_ERROR, ex.getMessage()); + } catch (ConcurrentOperationException ex) { + logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } catch (InsufficientCapacityException ex) { + StringBuilder message = new StringBuilder(ex.getMessage()); + if (ex instanceof InsufficientServerCapacityException) { + if (((InsufficientServerCapacityException)ex).isAffinityApplied()) { + message.append(", Please check the affinity groups provided, there may not be sufficient capacity to follow them"); + } + } + logger.info(String.format("%s: %s", message.toString(), ex.getLocalizedMessage())); + logger.debug(message.toString(), ex); + throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, message.toString()); + } + + if (vm != null) { + UserVmResponse response = _responseGenerator.createUserVmResponse(getResponseView(), "virtualmachine", vm).get(0); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to deploy vm uuid:"+getEntityUuid()); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index afd23cfd8715..dc1ca3583cc3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -16,87 +16,40 @@ // under the License. package org.apache.cloudstack.api.command.user.vm; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.stream.Stream; -import javax.annotation.Nonnull; - -import org.apache.cloudstack.acl.RoleType; -import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; -import org.apache.cloudstack.api.ApiArgValidator; -import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.BaseAsyncCreateCustomIdCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.command.user.UserCmd; -import org.apache.cloudstack.api.response.DiskOfferingResponse; -import org.apache.cloudstack.api.response.DomainResponse; -import org.apache.cloudstack.api.response.HostResponse; -import org.apache.cloudstack.api.response.NetworkResponse; -import org.apache.cloudstack.api.response.ProjectResponse; -import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.SnapshotResponse; import org.apache.cloudstack.api.response.TemplateResponse; -import org.apache.cloudstack.api.response.UserDataResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.VolumeResponse; -import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.vm.lease.VMLeaseManager; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.EnumUtils; -import org.apache.commons.lang3.StringUtils; -import com.cloud.agent.api.LogLevel; -import com.cloud.event.EventTypes; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InsufficientServerCapacityException; -import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.network.Network; -import com.cloud.network.Network.IpAddresses; -import com.cloud.offering.DiskOffering; -import com.cloud.template.VirtualMachineTemplate; import com.cloud.uservm.UserVm; import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.net.Dhcp; -import com.cloud.utils.net.NetUtils; import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VmDetailConstants; @APICommand(name = "deployVirtualMachine", description = "Creates and automatically starts a virtual machine based on a service offering, disk offering, and template.", responseObject = UserVmResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityGroupAction, UserCmd { - - private static final String s_name = "deployvirtualmachineresponse"; +public class DeployVMCmd extends BaseDeployVMCmd { ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, required = true, description = "availability zone for the virtual machine") - private Long zoneId; - @ACL @Parameter(name = ApiConstants.SERVICE_OFFERING_ID, type = CommandType.UUID, entityType = ServiceOfferingResponse.class, required = true, description = "the ID of the service offering for the virtual machine") private Long serviceOfferingId; @@ -105,672 +58,24 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG @Parameter(name = ApiConstants.TEMPLATE_ID, type = CommandType.UUID, entityType = TemplateResponse.class, description = "the ID of the template for the virtual machine") private Long templateId; - @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "host name for the virtual machine", validations = {ApiArgValidator.RFCComplianceDomainName}) - private String name; - - @Parameter(name = ApiConstants.DISPLAY_NAME, type = CommandType.STRING, description = "an optional user generated name for the virtual machine") - private String displayName; - - @Parameter(name=ApiConstants.PASSWORD, type=CommandType.STRING, description="The password of the virtual machine. If null, a random password will be generated for the VM.", - since="4.19.0.0") - protected String password; - - //Owner information - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the virtual machine. Must be used with domainId.") - private String accountName; - - @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used. If account is NOT provided then virtual machine will be assigned to the caller account and domain.") - private Long domainId; - - //Network information - //@ACL(accessType = AccessType.UseEntry) - @Parameter(name = ApiConstants.NETWORK_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = NetworkResponse.class, description = "list of network ids used by virtual machine. Can't be specified with ipToNetworkList parameter") - private List networkIds; - - @Parameter(name = ApiConstants.BOOT_TYPE, type = CommandType.STRING, required = false, description = "Guest VM Boot option either custom[UEFI] or default boot [BIOS]. Not applicable with VMware if the template is marked as deploy-as-is, as we honour what is defined in the template.", since = "4.14.0.0") - private String bootType; - - @Parameter(name = ApiConstants.BOOT_MODE, type = CommandType.STRING, required = false, description = "Boot Mode [Legacy] or [Secure] Applicable when Boot Type Selected is UEFI, otherwise Legacy only for BIOS. Not applicable with VMware if the template is marked as deploy-as-is, as we honour what is defined in the template.", since = "4.14.0.0") - private String bootMode; - - @Parameter(name = ApiConstants.BOOT_INTO_SETUP, type = CommandType.BOOLEAN, required = false, description = "Boot into hardware setup or not (ignored if startVm = false, only valid for vmware)", since = "4.15.0.0") - private Boolean bootIntoSetup; - - //DataDisk information - @ACL - @Parameter(name = ApiConstants.DISK_OFFERING_ID, type = CommandType.UUID, entityType = DiskOfferingResponse.class, description = "the ID of the disk offering for the virtual machine. If the template is of ISO format," - + " the diskOfferingId is for the root disk volume. Otherwise this parameter is used to indicate the " - + "offering for the data disk volume. If the templateId parameter passed is from a Template object," - + " the diskOfferingId refers to a DATA Disk Volume created. If the templateId parameter passed is " - + "from an ISO object, the diskOfferingId refers to a ROOT Disk Volume created.") - private Long diskOfferingId; - - @Parameter(name = ApiConstants.SIZE, type = CommandType.LONG, description = "the arbitrary size for the DATADISK volume. Mutually exclusive with diskOfferingId") - private Long size; - - @Parameter(name = ApiConstants.ROOT_DISK_SIZE, - type = CommandType.LONG, - description = "Optional field to resize root disk on deploy. Value is in GB. Only applies to template-based deployments. Analogous to details[0].rootdisksize, which takes precedence over this parameter if both are provided", - since = "4.4") - private Long rootdisksize; - - @Parameter(name = ApiConstants.GROUP, type = CommandType.STRING, description = "an optional group for the virtual machine") - private String group; - - @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, description = "the hypervisor on which to deploy the virtual machine. " - + "The parameter is required and respected only when hypervisor info is not set on the ISO/Template passed to the call") - private String hypervisor; - - @Parameter(name = ApiConstants.USER_DATA, type = CommandType.STRING, - description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " + - "This binary data must be base64 encoded before adding it to the request. " + - "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " + - "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " + - "You also need to change vm.userdata.max.length value", - length = 1048576) - private String userData; - - @Parameter(name = ApiConstants.USER_DATA_ID, type = CommandType.UUID, entityType = UserDataResponse.class, description = "the ID of the Userdata", since = "4.18") - private Long userdataId; - - @Parameter(name = ApiConstants.USER_DATA_DETAILS, type = CommandType.MAP, description = "used to specify the parameters values for the variables in userdata.", since = "4.18") - private Map userdataDetails; - - @Deprecated - @Parameter(name = ApiConstants.SSH_KEYPAIR, type = CommandType.STRING, description = "name of the ssh key pair used to login to the virtual machine") - private String sshKeyPairName; - - @Parameter(name = ApiConstants.SSH_KEYPAIRS, type = CommandType.LIST, collectionType = CommandType.STRING, since="4.17", description = "names of the ssh key pairs used to login to the virtual machine") - private List sshKeyPairNames; - - @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, description = "destination Host ID to deploy the VM to - parameter available for root admin only") - private Long hostId; - - @ACL - @Parameter(name = ApiConstants.SECURITY_GROUP_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = SecurityGroupResponse.class, description = "comma separated list of security groups id that going to be applied to the virtual machine. " - + "Should be passed only when vm is created from a zone with Basic Network support." + " Mutually exclusive with securitygroupnames parameter") - private List securityGroupIdList; - - @ACL - @Parameter(name = ApiConstants.SECURITY_GROUP_NAMES, type = CommandType.LIST, collectionType = CommandType.STRING, entityType = SecurityGroupResponse.class, description = "comma separated list of security groups names that going to be applied to the virtual machine." - + " Should be passed only when vm is created from a zone with Basic Network support. " + "Mutually exclusive with securitygroupids parameter") - private List securityGroupNameList; - - @Parameter(name = ApiConstants.IP_NETWORK_LIST, type = CommandType.MAP, description = "ip to network mapping. Can't be specified with networkIds parameter." - + " Example: iptonetworklist[0].ip=10.10.10.11&iptonetworklist[0].ipv6=fc00:1234:5678::abcd&iptonetworklist[0].networkid=uuid&iptonetworklist[0].mac=aa:bb:cc:dd:ee::ff - requests to use ip 10.10.10.11 in network id=uuid") - private Map ipToNetworkList; - - @Parameter(name = ApiConstants.IP_ADDRESS, type = CommandType.STRING, description = "the ip address for default vm's network") - private String ipAddress; - - @Parameter(name = ApiConstants.IP6_ADDRESS, type = CommandType.STRING, description = "the ipv6 address for default vm's network") - private String ip6Address; - - @Parameter(name = ApiConstants.MAC_ADDRESS, type = CommandType.STRING, description = "the mac address for default vm's network") - private String macAddress; - - @Parameter(name = ApiConstants.KEYBOARD, type = CommandType.STRING, description = "an optional keyboard device type for the virtual machine. valid value can be one of de,de-ch,es,fi,fr,fr-be,fr-ch,is,it,jp,nl-be,no,pt,uk,us") - private String keyboard; - - @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "Deploy vm for the project") - private Long projectId; - - @Parameter(name = ApiConstants.START_VM, type = CommandType.BOOLEAN, description = "true if start vm after creating; defaulted to true if not specified") - private Boolean startVm; - - @ACL - @Parameter(name = ApiConstants.AFFINITY_GROUP_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = AffinityGroupResponse.class, description = "comma separated list of affinity groups id that are going to be applied to the virtual machine." - + " Mutually exclusive with affinitygroupnames parameter") - private List affinityGroupIdList; - - @ACL - @Parameter(name = ApiConstants.AFFINITY_GROUP_NAMES, type = CommandType.LIST, collectionType = CommandType.STRING, entityType = AffinityGroupResponse.class, description = "comma separated list of affinity groups names that are going to be applied to the virtual machine." - + "Mutually exclusive with affinitygroupids parameter") - private List affinityGroupNameList; - - @Parameter(name = ApiConstants.DISPLAY_VM, type = CommandType.BOOLEAN, since = "4.2", description = "an optional field, whether to the display the vm to the end user or not.", authorized = {RoleType.Admin}) - private Boolean displayVm; - - @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, since = "4.3", description = "used to specify the custom parameters. 'extraconfig' is not allowed to be passed in details") - private Map details; - - @Parameter(name = ApiConstants.DEPLOYMENT_PLANNER, type = CommandType.STRING, description = "Deployment planner to use for vm allocation. Available to ROOT admin only", since = "4.4", authorized = { RoleType.Admin }) - private String deploymentPlanner; - - @Parameter(name = ApiConstants.DHCP_OPTIONS_NETWORK_LIST, type = CommandType.MAP, description = "DHCP options which are passed to the VM on start up" - + " Example: dhcpoptionsnetworklist[0].dhcp:114=url&dhcpoptionsetworklist[0].networkid=networkid&dhcpoptionsetworklist[0].dhcp:66=www.test.com") - private Map dhcpOptionsNetworkList; - - @Parameter(name = ApiConstants.DATADISK_OFFERING_LIST, type = CommandType.MAP, since = "4.11", description = "datadisk template to disk-offering mapping;" + - " an optional parameter used to create additional data disks from datadisk templates; can't be specified with diskOfferingId parameter") - private Map dataDiskTemplateToDiskOfferingList; - - @Parameter(name = ApiConstants.EXTRA_CONFIG, type = CommandType.STRING, since = "4.12", description = "an optional URL encoded string that can be passed to the virtual machine upon successful deployment", length = 5120) - private String extraConfig; - - @Parameter(name = ApiConstants.COPY_IMAGE_TAGS, type = CommandType.BOOLEAN, since = "4.13", description = "if true the image tags (if any) will be copied to the VM, default value is false") - private Boolean copyImageTags; - - @Parameter(name = ApiConstants.PROPERTIES, type = CommandType.MAP, since = "4.15", - description = "used to specify the vApp properties.") - @LogLevel(LogLevel.Log4jLevel.Off) - private Map vAppProperties; - - @Parameter(name = ApiConstants.NIC_NETWORK_LIST, type = CommandType.MAP, since = "4.15", - description = "VMware only: used to specify network mapping of a vApp VMware template registered \"as-is\"." + - " Example nicnetworklist[0].ip=Nic-101&nicnetworklist[0].network=uuid") - @LogLevel(LogLevel.Log4jLevel.Off) - private Map vAppNetworks; - - @Parameter(name = ApiConstants.DYNAMIC_SCALING_ENABLED, type = CommandType.BOOLEAN, since = "4.16", - description = "true if virtual machine needs to be dynamically scalable") - protected Boolean dynamicScalingEnabled; - - @Parameter(name = ApiConstants.OVERRIDE_DISK_OFFERING_ID, type = CommandType.UUID, since = "4.17", entityType = DiskOfferingResponse.class, description = "the ID of the disk offering for the virtual machine to be used for root volume instead of the disk offering mapped in service offering." + - "In case of virtual machine deploying from ISO, then the diskofferingid specified for root volume is ignored and uses this override disk offering id") - private Long overrideDiskOfferingId; - - @Parameter(name = ApiConstants.IOTHREADS_ENABLED, type = CommandType.BOOLEAN, required = false, - description = "IOThreads are dedicated event loop threads for supported disk devices to perform block I/O requests in order to improve scalability especially on an SMP host/guest with many LUNs.") - private Boolean iothreadsEnabled; - - @Parameter(name = ApiConstants.IO_DRIVER_POLICY, type = CommandType.STRING, description = "Controls specific policies on IO") - private String ioDriverPolicy; - - @Parameter(name = ApiConstants.NIC_MULTIQUEUE_NUMBER, type = CommandType.INTEGER, since = "4.18", - description = "The number of queues for multiqueue NICs.") - private Integer nicMultiqueueNumber; - - @Parameter(name = ApiConstants.NIC_PACKED_VIRTQUEUES_ENABLED, type = CommandType.BOOLEAN, since = "4.18", - description = "Enable packed virtqueues or not.") - private Boolean nicPackedVirtQueues; - - @Parameter(name = ApiConstants.INSTANCE_LEASE_DURATION, type = CommandType.INTEGER, since = "4.21.0", - description = "Number of days instance is leased for.") - private Integer leaseDuration; - - @Parameter(name = ApiConstants.INSTANCE_LEASE_EXPIRY_ACTION, type = CommandType.STRING, since = "4.21.0", - description = "Lease expiry action, valid values are STOP and DESTROY") - private String leaseExpiryAction; - @Parameter(name = ApiConstants.VOLUME_ID, type = CommandType.UUID, entityType = VolumeResponse.class, since = "4.21") private Long volumeId; @Parameter(name = ApiConstants.SNAPSHOT_ID, type = CommandType.UUID, entityType = SnapshotResponse.class, since = "4.21") private Long snapshotId; - @Parameter(name = ApiConstants.EXTERNAL_DETAILS, - type = CommandType.MAP, - description = "Details in key/value pairs using format externaldetails[i].keyname=keyvalue. Example: externaldetails[0].server.type=typevalue", - since = "4.21.0") - protected Map externalDetails; - ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// - public String getAccountName() { - if (accountName == null) { - return CallContext.current().getCallingAccount().getAccountName(); - } - return accountName; - } - - public Long getDiskOfferingId() { - return diskOfferingId; - } - - public String getDeploymentPlanner() { - return deploymentPlanner; - } - - public String getDisplayName() { - return displayName; - } - - public Long getDomainId() { - if (domainId == null) { - return CallContext.current().getCallingAccount().getDomainId(); - } - return domainId; - } - - public ApiConstants.BootType getBootType() { - if (StringUtils.isNotBlank(bootType)) { - try { - String type = bootType.trim().toUpperCase(); - return ApiConstants.BootType.valueOf(type); - } catch (IllegalArgumentException e) { - String errMesg = "Invalid bootType " + bootType + "Specified for vm " + getName() - + " Valid values are: " + Arrays.toString(ApiConstants.BootType.values()); - logger.warn(errMesg); - throw new InvalidParameterValueException(errMesg); - } - } - return null; - } - - public Map getDetails() { - Map customparameterMap = convertDetailsToMap(details); - - if (getBootType() != null) { - customparameterMap.put(getBootType().toString(), getBootMode().toString()); - } - - if (rootdisksize != null && !customparameterMap.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) { - customparameterMap.put(VmDetailConstants.ROOT_DISK_SIZE, rootdisksize.toString()); - } - - IoDriverPolicy ioPolicy = getIoDriverPolicy(); - if (ioPolicy != null) { - customparameterMap.put(VmDetailConstants.IO_POLICY, ioPolicy.toString()); - } - - if (BooleanUtils.toBoolean(iothreadsEnabled)) { - customparameterMap.put(VmDetailConstants.IOTHREADS, BooleanUtils.toStringTrueFalse(iothreadsEnabled)); - } - - if (nicMultiqueueNumber != null) { - customparameterMap.put(VmDetailConstants.NIC_MULTIQUEUE_NUMBER, nicMultiqueueNumber.toString()); - } - - if (BooleanUtils.toBoolean(nicPackedVirtQueues)) { - customparameterMap.put(VmDetailConstants.NIC_PACKED_VIRTQUEUES_ENABLED, BooleanUtils.toStringTrueFalse(nicPackedVirtQueues)); - } - - if (MapUtils.isNotEmpty(externalDetails)) { - customparameterMap.putAll(getExternalDetails()); - } - - return customparameterMap; - } - - public Map getExternalDetails() { - return convertExternalDetailsToMap(externalDetails); - } - - public ApiConstants.BootMode getBootMode() { - if (StringUtils.isNotBlank(bootMode)) { - try { - String mode = bootMode.trim().toUpperCase(); - return ApiConstants.BootMode.valueOf(mode); - } catch (IllegalArgumentException e) { - String msg = String.format("Invalid %s: %s specified for VM: %s. Valid values are: %s", - ApiConstants.BOOT_MODE, bootMode, getName(), Arrays.toString(ApiConstants.BootMode.values())); - logger.error(msg); - throw new InvalidParameterValueException(msg); - } - } - if (ApiConstants.BootType.UEFI.equals(getBootType())) { - String msg = String.format("%s must be specified for the VM with boot type: %s. Valid values are: %s", - ApiConstants.BOOT_MODE, getBootType(), Arrays.toString(ApiConstants.BootMode.values())); - logger.error(msg); - throw new InvalidParameterValueException(msg); - } - return null; - } - - public Map getVmProperties() { - Map map = new HashMap<>(); - if (MapUtils.isNotEmpty(vAppProperties)) { - Collection parameterCollection = vAppProperties.values(); - Iterator iterator = parameterCollection.iterator(); - while (iterator.hasNext()) { - HashMap entry = (HashMap)iterator.next(); - map.put(entry.get("key"), entry.get("value")); - } - } - return map; - } - - public Map getVmNetworkMap() { - Map map = new HashMap<>(); - if (MapUtils.isNotEmpty(vAppNetworks)) { - Collection parameterCollection = vAppNetworks.values(); - Iterator iterator = parameterCollection.iterator(); - while (iterator.hasNext()) { - HashMap entry = (HashMap) iterator.next(); - Integer nic; - try { - nic = Integer.valueOf(entry.get(VmDetailConstants.NIC)); - } catch (NumberFormatException nfe) { - nic = null; - } - String networkUuid = entry.get(VmDetailConstants.NETWORK); - if (logger.isTraceEnabled()) { - logger.trace(String.format("nic, '%s', goes on net, '%s'", nic, networkUuid)); - } - if (nic == null || StringUtils.isEmpty(networkUuid) || _entityMgr.findByUuid(Network.class, networkUuid) == null) { - throw new InvalidParameterValueException(String.format("Network ID: %s for NIC ID: %s is invalid", networkUuid, nic)); - } - map.put(nic, _entityMgr.findByUuid(Network.class, networkUuid).getId()); - } - } - return map; - } - - public String getGroup() { - return group; - } - - public HypervisorType getHypervisor() { - return HypervisorType.getType(hypervisor); - } - - public Boolean isDisplayVm() { - return displayVm; - } - - @Override - public boolean isDisplay() { - if(displayVm == null) - return true; - else - return displayVm; - } - - public List getSecurityGroupNameList() { - return securityGroupNameList; - } - - public List getSecurityGroupIdList() { - return securityGroupIdList; - } - public Long getServiceOfferingId() { return serviceOfferingId; } - public Long getSize() { - return size; - } - public Long getTemplateId() { return templateId; } - public String getUserData() { - return userData; - } - - public Long getUserdataId() { - return userdataId; - } - - public Map getUserdataDetails() { - return convertDetailsToMap(userdataDetails); - } - - public Long getZoneId() { - return zoneId; - } - - public String getPassword() { - return password; - } - - public Integer getLeaseDuration() { - return leaseDuration; - } - - public VMLeaseManager.ExpiryAction getLeaseExpiryAction() { - if (StringUtils.isBlank(leaseExpiryAction)) { - return null; - } - VMLeaseManager.ExpiryAction action = EnumUtils.getEnumIgnoreCase(VMLeaseManager.ExpiryAction.class, leaseExpiryAction); - if (action == null) { - throw new InvalidParameterValueException("Invalid value configured for leaseexpiryaction, valid values are: " + - com.cloud.utils.EnumUtils.listValues(VMLeaseManager.ExpiryAction.values())); - } - return action; - } - - public List getNetworkIds() { - if (MapUtils.isNotEmpty(vAppNetworks)) { - if (CollectionUtils.isNotEmpty(networkIds) || ipAddress != null || getIp6Address() != null || MapUtils.isNotEmpty(ipToNetworkList)) { - throw new InvalidParameterValueException(String.format("%s can't be specified along with %s, %s, %s", ApiConstants.NIC_NETWORK_LIST, ApiConstants.NETWORK_IDS, ApiConstants.IP_ADDRESS, ApiConstants.IP_NETWORK_LIST)); - } else { - return new ArrayList<>(); - } - } - if (ipToNetworkList != null && !ipToNetworkList.isEmpty()) { - if ((networkIds != null && !networkIds.isEmpty()) || ipAddress != null || getIp6Address() != null) { - throw new InvalidParameterValueException("ipToNetworkMap can't be specified along with networkIds or ipAddress"); - } else { - List networks = new ArrayList(); - networks.addAll(getIpToNetworkMap().keySet()); - return networks; - } - } - return networkIds; - } - - public String getName() { - return name; - } - - public List getSSHKeyPairNames() { - List sshKeyPairs = new ArrayList(); - if(sshKeyPairNames != null) { - sshKeyPairs = sshKeyPairNames; - } - if(sshKeyPairName != null && !sshKeyPairName.isEmpty()) { - sshKeyPairs.add(sshKeyPairName); - } - return sshKeyPairs; - } - - public Long getHostId() { - return hostId; - } - - public boolean getStartVm() { - return startVm == null ? true : startVm; - } - - public Map getIpToNetworkMap() { - if ((networkIds != null || ipAddress != null || getIp6Address() != null) && ipToNetworkList != null) { - throw new InvalidParameterValueException("NetworkIds and ipAddress can't be specified along with ipToNetworkMap parameter"); - } - LinkedHashMap ipToNetworkMap = null; - if (ipToNetworkList != null && !ipToNetworkList.isEmpty()) { - ipToNetworkMap = new LinkedHashMap(); - Collection ipsCollection = ipToNetworkList.values(); - Iterator iter = ipsCollection.iterator(); - while (iter.hasNext()) { - HashMap ips = (HashMap)iter.next(); - Long networkId = getNetworkIdFomIpMap(ips); - IpAddresses addrs = getIpAddressesFromIpMap(ips); - ipToNetworkMap.put(networkId, addrs); - } - } - - return ipToNetworkMap; - } - - @Nonnull - private IpAddresses getIpAddressesFromIpMap(HashMap ips) { - String requestedIp = ips.get("ip"); - String requestedIpv6 = ips.get("ipv6"); - String requestedMac = ips.get("mac"); - if (requestedIpv6 != null) { - requestedIpv6 = NetUtils.standardizeIp6Address(requestedIpv6); - } - if (requestedMac != null) { - if(!NetUtils.isValidMac(requestedMac)) { - throw new InvalidParameterValueException("Mac address is not valid: " + requestedMac); - } else if(!NetUtils.isUnicastMac(requestedMac)) { - throw new InvalidParameterValueException("Mac address is not unicast: " + requestedMac); - } - requestedMac = NetUtils.standardizeMacAddress(requestedMac); - } - return new IpAddresses(requestedIp, requestedIpv6, requestedMac); - } - - @Nonnull - private Long getNetworkIdFomIpMap(HashMap ips) { - Long networkId; - final String networkid = ips.get("networkid"); - Network network = _networkService.getNetwork(networkid); - if (network != null) { - networkId = network.getId(); - } else { - try { - networkId = Long.parseLong(networkid); - } catch (NumberFormatException e) { - throw new InvalidParameterValueException("Unable to translate and find entity with networkId: " + networkid); - } - } - return networkId; - } - - public String getIpAddress() { - return ipAddress; - } - - public String getIp6Address() { - if (ip6Address == null) { - return null; - } - return NetUtils.standardizeIp6Address(ip6Address); - } - - - public String getMacAddress() { - if (macAddress == null) { - return null; - } - if(!NetUtils.isValidMac(macAddress)) { - throw new InvalidParameterValueException("Mac address is not valid: " + macAddress); - } else if(!NetUtils.isUnicastMac(macAddress)) { - throw new InvalidParameterValueException("Mac address is not unicast: " + macAddress); - } - return NetUtils.standardizeMacAddress(macAddress); - } - - public List getAffinityGroupIdList() { - if (affinityGroupNameList != null && affinityGroupIdList != null) { - throw new InvalidParameterValueException("affinitygroupids parameter is mutually exclusive with affinitygroupnames parameter"); - } - - // transform group names to ids here - if (affinityGroupNameList != null) { - List affinityGroupIds = new ArrayList(); - for (String groupName : affinityGroupNameList) { - Long groupId = _responseGenerator.getAffinityGroupId(groupName, getEntityOwnerId()); - if (groupId == null) { - throw new InvalidParameterValueException("Unable to find affinity group by name " + groupName); - } else { - affinityGroupIds.add(groupId); - } - } - return affinityGroupIds; - } else { - return affinityGroupIdList; - } - } - - public String getKeyboard() { - // TODO Auto-generated method stub - return keyboard; - } - - public Map> getDhcpOptionsMap() { - Map> dhcpOptionsMap = new HashMap<>(); - if (dhcpOptionsNetworkList != null && !dhcpOptionsNetworkList.isEmpty()) { - - Collection> paramsCollection = this.dhcpOptionsNetworkList.values(); - for (Map dhcpNetworkOptions : paramsCollection) { - String networkId = dhcpNetworkOptions.get(ApiConstants.NETWORK_ID); - - if (networkId == null) { - throw new IllegalArgumentException("No networkid specified when providing extra dhcp options."); - } - - Map dhcpOptionsForNetwork = new HashMap<>(); - dhcpOptionsMap.put(networkId, dhcpOptionsForNetwork); - - for (String key : dhcpNetworkOptions.keySet()) { - if (key.startsWith(ApiConstants.DHCP_PREFIX)) { - int dhcpOptionValue = Integer.parseInt(key.replaceFirst(ApiConstants.DHCP_PREFIX, "")); - dhcpOptionsForNetwork.put(dhcpOptionValue, dhcpNetworkOptions.get(key)); - } else if (!key.equals(ApiConstants.NETWORK_ID)) { - Dhcp.DhcpOptionCode dhcpOptionEnum = Dhcp.DhcpOptionCode.valueOfString(key); - dhcpOptionsForNetwork.put(dhcpOptionEnum.getCode(), dhcpNetworkOptions.get(key)); - } - } - - } - } - - return dhcpOptionsMap; - } - - public Map getDataDiskTemplateToDiskOfferingMap() { - if (diskOfferingId != null && dataDiskTemplateToDiskOfferingList != null) { - throw new InvalidParameterValueException("diskofferingid parameter can't be specified along with datadisktemplatetodiskofferinglist parameter"); - } - if (MapUtils.isEmpty(dataDiskTemplateToDiskOfferingList)) { - return new HashMap(); - } - - HashMap dataDiskTemplateToDiskOfferingMap = new HashMap(); - for (Object objDataDiskTemplates : dataDiskTemplateToDiskOfferingList.values()) { - HashMap dataDiskTemplates = (HashMap) objDataDiskTemplates; - Long dataDiskTemplateId; - DiskOffering dataDiskOffering = null; - VirtualMachineTemplate dataDiskTemplate= _entityMgr.findByUuid(VirtualMachineTemplate.class, dataDiskTemplates.get("datadisktemplateid")); - if (dataDiskTemplate == null) { - dataDiskTemplate = _entityMgr.findById(VirtualMachineTemplate.class, dataDiskTemplates.get("datadisktemplateid")); - if (dataDiskTemplate == null) - throw new InvalidParameterValueException("Unable to translate and find entity with datadisktemplateid " + dataDiskTemplates.get("datadisktemplateid")); - } - dataDiskTemplateId = dataDiskTemplate.getId(); - dataDiskOffering = _entityMgr.findByUuid(DiskOffering.class, dataDiskTemplates.get("diskofferingid")); - if (dataDiskOffering == null) { - dataDiskOffering = _entityMgr.findById(DiskOffering.class, dataDiskTemplates.get("diskofferingid")); - if (dataDiskOffering == null) - throw new InvalidParameterValueException("Unable to translate and find entity with diskofferingId " + dataDiskTemplates.get("diskofferingid")); - } - dataDiskTemplateToDiskOfferingMap.put(dataDiskTemplateId, dataDiskOffering); - } - return dataDiskTemplateToDiskOfferingMap; - } - - public String getExtraConfig() { - return extraConfig; - } - - public boolean getCopyImageTags() { - return copyImageTags == null ? false : copyImageTags; - } - - public Boolean getBootIntoSetup() { - return bootIntoSetup; - } - - public boolean isDynamicScalingEnabled() { - return dynamicScalingEnabled == null ? true : dynamicScalingEnabled; - } - - public Long getOverrideDiskOfferingId() { - return overrideDiskOfferingId; - } - - public ApiConstants.IoDriverPolicy getIoDriverPolicy() { - if (StringUtils.isNotBlank(ioDriverPolicy)) { - try { - String policyType = ioDriverPolicy.trim().toUpperCase(); - return ApiConstants.IoDriverPolicy.valueOf(policyType); - } catch (IllegalArgumentException e) { - String errMesg = String.format("Invalid io policy %s specified for vm %s. Valid values are: %s", ioDriverPolicy, getName(), Arrays.toString(ApiConstants.IoDriverPolicy.values())); - logger.warn(errMesg); - throw new InvalidParameterValueException(errMesg); - } - } - return null; - } - public Long getVolumeId() { return volumeId; } @@ -782,57 +87,6 @@ public Long getSnapshotId() { public boolean isVolumeOrSnapshotProvided() { return volumeId != null || snapshotId != null; } - ///////////////////////////////////////////////////// - /////////////// API Implementation/////////////////// - ///////////////////////////////////////////////////// - - @Override - public String getCommandName() { - return s_name; - } - - public static String getResultObjectName() { - return "virtualmachine"; - } - - @Override - public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); - if (accountId == null) { - return CallContext.current().getCallingAccount().getId(); - } - - return accountId; - } - - @Override - public String getEventType() { - return EventTypes.EVENT_VM_CREATE; - } - - @Override - public String getCreateEventType() { - return EventTypes.EVENT_VM_CREATE; - } - - @Override - public String getCreateEventDescription() { - return "creating Vm"; - } - - @Override - public String getEventDescription() { - if(getStartVm()) { - return "starting Vm. Vm Id: " + getEntityUuid(); - } - return "deploying Vm. Vm Id: " + getEntityUuid(); - } - - @Override - public ApiCommandResourceType getApiResourceType() { - return ApiCommandResourceType.VirtualMachine; - } - @Override public void execute() { UserVm result; @@ -875,7 +129,6 @@ public void execute() { } } - @Override public void create() throws ResourceAllocationException { if (Stream.of(templateId, snapshotId, volumeId).filter(Objects::nonNull).count() != 1) { diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupOfferingResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupOfferingResponse.java index 480ebcfb13d3..0e895fa4e967 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupOfferingResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupOfferingResponse.java @@ -41,6 +41,10 @@ public class BackupOfferingResponse extends BaseResponse { @Param(description = "description for the backup offering") private String description; + @SerializedName(ApiConstants.PROVIDER) + @Param(description = "provider name", since = "4.21.0") + private String provider; + @SerializedName(ApiConstants.EXTERNAL_ID) @Param(description = "external ID on the provider side") private String externalId; @@ -69,6 +73,10 @@ public void setExternalId(String externalId) { this.externalId = externalId; } + public void setProvider(String provider) { + this.provider = provider; + } + public void setName(String name) { this.name = name; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java index 63419680fea3..0ae558ac8033 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java @@ -26,6 +26,7 @@ import com.google.gson.annotations.SerializedName; import java.util.Date; +import java.util.Map; @EntityReference(value = Backup.class) public class BackupResponse extends BaseResponse { @@ -34,6 +35,14 @@ public class BackupResponse extends BaseResponse { @Param(description = "ID of the VM backup") private String id; + @SerializedName(ApiConstants.NAME) + @Param(description = "name of the backup", since = "4.21.0") + private String name; + + @SerializedName(ApiConstants.DESCRIPTION) + @Param(description = "description for the backup", since = "4.21.0") + private String description; + @SerializedName(ApiConstants.VIRTUAL_MACHINE_ID) @Param(description = "ID of the VM") private String vmId; @@ -102,6 +111,18 @@ public class BackupResponse extends BaseResponse { @Param(description = "zone name") private String zone; + @SerializedName(ApiConstants.VM_DETAILS) + @Param(description = "Lists the vm specific details for the backup", since = "4.21.0") + private Map vmDetails; + + @SerializedName(ApiConstants.INTERVAL_TYPE) + @Param(description = "Interval type of the backup", since = "4.21.0") + private String intervalType; + + @SerializedName(ApiConstants.BACKUP_VM_OFFERING_REMOVED) + @Param(description = "The backup offering corresponding to this backup was removed from the VM", since = "4.21.0") + private Boolean vmOfferingRemoved; + public String getId() { return id; } @@ -110,6 +131,22 @@ public void setId(String id) { this.id = id; } + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + public String getVmId() { return vmId; } @@ -245,4 +282,28 @@ public String getZone() { public void setZone(String zone) { this.zone = zone; } + + public Map getVmDetails() { + return vmDetails; + } + + public void setVmDetails(Map vmDetails) { + this.vmDetails = vmDetails; + } + + public String getIntervalType() { + return this.intervalType; + } + + public void setIntervalType(String intervalType) { + this.intervalType = intervalType; + } + + public Boolean getVmOfferingRemoved() { + return this.vmOfferingRemoved; + } + + public void setVmOfferingRemoved(Boolean vmOfferingRemoved) { + this.vmOfferingRemoved = vmOfferingRemoved; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java index 9584223563f8..d8c2980774d3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java @@ -60,6 +60,10 @@ public void setId(String id) { this.id = id; } + @SerializedName(ApiConstants.QUIESCE_VM) + @Param(description = "quiesce the instance before checkpointing the disks for backup") + private Boolean quiesceVM; + public String getVmName() { return vmName; } @@ -103,4 +107,8 @@ public void setTimezone(String timezone) { public void setMaxBackups(Integer maxBackups) { this.maxBackups = maxBackups; } + + public void setQuiesceVM(Boolean quiesceVM) { + this.quiesceVM = quiesceVM; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ObjectStoreResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ObjectStoreResponse.java index e4030799aa79..dcb93aaaf1d2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ObjectStoreResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ObjectStoreResponse.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.api.response; import com.cloud.serializer.Param; + +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.storage.object.ObjectStore; import com.google.gson.annotations.SerializedName; import org.apache.cloudstack.api.BaseResponseWithAnnotations; @@ -24,15 +26,15 @@ @EntityReference(value = ObjectStore.class) public class ObjectStoreResponse extends BaseResponseWithAnnotations { - @SerializedName("id") + @SerializedName(ApiConstants.ID) @Param(description = "the ID of the object store") private String id; - @SerializedName("name") + @SerializedName(ApiConstants.NAME) @Param(description = "the name of the object store") private String name; - @SerializedName("url") + @SerializedName(ApiConstants.URL) @Param(description = "the url of the object store") private String url; @@ -44,6 +46,10 @@ public class ObjectStoreResponse extends BaseResponseWithAnnotations { @Param(description = "the total size of the object store") private Long storageTotal; + @SerializedName("storageallocated") + @Param(description = "the allocated size of the object store") + private Long storageAllocated; + @SerializedName("storageused") @Param(description = "the object store currently used size") private Long storageUsed; @@ -96,6 +102,14 @@ public void setStorageTotal(Long storageTotal) { this.storageTotal = storageTotal; } + public Long getStorageAllocated() { + return storageAllocated; + } + + public void setStorageAllocated(Long storageAllocated) { + this.storageAllocated = storageAllocated; + } + public Long getStorageUsed() { return storageUsed; } diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index 53ac0ae960e6..951af9180e7f 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -19,6 +19,7 @@ import java.util.Date; import java.util.List; +import java.util.Map; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.api.Identity; @@ -63,6 +64,8 @@ class RestorePoint { private String id; private Date created; private String type; + private Long backupSize = 0L; + private Long dataSize = 0L; public RestorePoint(String id, Date created, String type) { this.id = id; @@ -70,6 +73,12 @@ public RestorePoint(String id, Date created, String type) { this.type = type; } + public RestorePoint(String id, Date created, String type, Long backupSize, Long dataSize) { + this(id, created, type); + this.backupSize = backupSize; + this.dataSize = dataSize; + } + public String getId() { return id; } @@ -93,6 +102,22 @@ public String getType() { public void setType(String type) { this.type = type; } + + public Long getBackupSize() { + return backupSize; + } + + public void setBackupSize(Long backupSize) { + this.backupSize = backupSize; + } + + public Long getDataSize() { + return dataSize; + } + + public void setDataSize(Long dataSize) { + this.dataSize = dataSize; + } } class VolumeInfo { @@ -100,12 +125,20 @@ class VolumeInfo { private Volume.Type type; private Long size; private String path; + private Long deviceId; + private String diskOfferingId; + private Long minIops; + private Long maxIops; - public VolumeInfo(String uuid, String path, Volume.Type type, Long size) { + public VolumeInfo(String uuid, String path, Volume.Type type, Long size, Long deviceId, String diskOfferingId, Long minIops, Long maxIops) { this.uuid = uuid; this.type = type; this.size = size; this.path = path; + this.deviceId = deviceId; + this.diskOfferingId = diskOfferingId; + this.minIops = minIops; + this.maxIops = maxIops; } public String getUuid() { @@ -128,13 +161,29 @@ public Long getSize() { return size; } + public Long getDeviceId() { + return deviceId; + } + + public String getDiskOfferingId() { + return diskOfferingId; + } + + public Long getMinIops() { + return minIops; + } + + public Long getMaxIops() { + return maxIops; + } + @Override public String toString() { - return StringUtils.join(":", uuid, path, type, size); + return StringUtils.join(":", uuid, path, type, size, deviceId, diskOfferingId, minIops, maxIops); } } - long getVmId(); + Long getVmId(); long getBackupOfferingId(); String getExternalId(); String getType(); @@ -142,7 +191,12 @@ public String toString() { Backup.Status getStatus(); Long getSize(); Long getProtectedSize(); + void setName(String name); + String getDescription(); + void setDescription(String description); List getBackedUpVolumes(); long getZoneId(); + Map getDetails(); + String getDetail(String name); Long getBackupScheduleId(); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index 2ed4d5af5d1d..c4b92fc9e05c 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -18,20 +18,29 @@ package org.apache.cloudstack.backup; import java.util.List; +import java.util.Map; +import com.cloud.capacity.Capacity; import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; +import org.apache.cloudstack.api.command.user.backup.CreateBackupCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.DeleteBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.ListBackupOfferingsCmd; import org.apache.cloudstack.api.command.user.backup.ListBackupsCmd; +import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.network.Network; +import com.cloud.storage.Volume; import com.cloud.utils.Pair; import com.cloud.utils.component.Manager; import com.cloud.utils.component.PluggableService; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmDiskInfo; /** * Backup and Recover Manager Interface @@ -106,6 +115,14 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer ConfigKey.Scope.Global, null); + ConfigKey BackupStorageCapacityThreshold = new ConfigKey<>("Alert", Float.class, + "zone.backupStorage.capacity.notificationthreshold", + "0.75", + "Percentage (as a value between 0 and 1) of backup storage utilization above which alerts will be sent about low storage available.", + true, + ConfigKey.Scope.Zone, + null); + /** * List backup provider offerings * @param zoneId zone id @@ -168,11 +185,11 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer /** * Creates backup of a VM - * @param vmId Virtual Machine ID + * @param cmd CreateBackupCmd * @param job The async job associated with the backup retention * @return returns operation success */ - boolean createBackup(final Long vmId, Object job) throws ResourceAllocationException; + boolean createBackup(CreateBackupCmd cmd, Object job) throws ResourceAllocationException; /** * List existing backups for a VM @@ -184,6 +201,15 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer */ boolean restoreBackup(final Long backupId); + Map getIpToNetworkMapFromBackup(Backup backup, boolean preserveIps, List networkIds); + + Boolean canCreateInstanceFromBackup(Long backupId); + + /** + * Restore a backup to a new Instance + */ + boolean restoreBackupToVM(Long backupId, Long vmId) throws ResourceUnavailableException; + /** * Restore a backed up volume and attach it to a VM */ @@ -197,5 +223,25 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer */ boolean deleteBackup(final Long backupId, final Boolean forced); + void validateBackupForZone(Long zoneId); + BackupOffering updateBackupOffering(UpdateBackupOfferingCmd updateBackupOfferingCmd); + + VmDiskInfo getRootDiskInfoFromBackup(Backup backup); + + List getDataDiskInfoListFromBackup(Backup backup); + + void checkVmDisksSizeAgainstBackup(List vmDiskInfoList, Backup backup); + + Map getBackupDetailsFromVM(VirtualMachine vm); + + String createVolumeInfoFromVolumes(List vmVolumes); + + String getBackupNameFromVM(VirtualMachine vm); + + BackupResponse createBackupResponse(Backup backup, Boolean listVmDetails); + + Capacity getBackupStorageUsedStats(Long zoneId); + + void checkAndRemoveBackupOfferingBeforeExpunge(VirtualMachine vm); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index 39582b0e423c..1eb36f895565 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -17,7 +17,6 @@ package org.apache.cloudstack.backup; import java.util.List; -import java.util.Map; import com.cloud.utils.Pair; import com.cloud.vm.VirtualMachine; @@ -71,10 +70,12 @@ public interface BackupProvider { /** * Starts and creates an adhoc backup process * for a previously registered VM backup - * @param vm the machine to make a backup of + * + * @param vm the machine to make a backup of + * @param quiesceVM instance will be quiesced for checkpointing for backup. Applicable only to NAS plugin. * @return the result and {code}Backup{code} {code}Object{code} */ - Pair takeBackup(VirtualMachine vm); + Pair takeBackup(VirtualMachine vm, Boolean quiesceVM); /** * Delete an existing backup @@ -84,6 +85,8 @@ public interface BackupProvider { */ boolean deleteBackup(Backup backup, boolean forced); + boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid); + /** * Restore VM from backup */ @@ -92,27 +95,44 @@ public interface BackupProvider { /** * Restore a volume from a backup */ - Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid, Pair vmNameAndState); + Pair restoreBackedUpVolume(Backup backup, Backup.VolumeInfo backupVolumeInfo, String hostIp, String dataStoreUuid, Pair vmNameAndState); /** - * Returns backup metrics for a list of VMs in a zone + * Syncs backup metrics (backup size, protected size) from the plugin and stores it within the provider * @param zoneId the zone for which to return metrics - * @param vms a list of machines to get measurements for - * @return a map of machine -> backup metrics */ - Map getBackupMetrics(Long zoneId, List vms); + void syncBackupMetrics(Long zoneId); /** - * This method should TODO - * @param vm the machine to get restore point for + * Returns a list of Backup.RestorePoint + * @param vm the machine to get the restore points for */ List listRestorePoints(VirtualMachine vm); /** - * This method should TODO + * Creates and returns an entry in the backups table by getting the information from restorePoint and vm. + * * @param restorePoint the restore point to create a backup for - * @param vm The machine for which to create a backup - * @param metric the metric object to update with the new backup data + * @param vm The machine for which to create a backup + */ + Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm); + + /** + * Returns if the backup provider supports creating new instance from backup */ - Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric); + boolean supportsInstanceFromBackup(); + + /** + * Returns the backup storage usage (Used, Total) for a backup provider + * @param zoneId the zone for which to return metrics + * @return a pair of Used size and Total size for the backup storage + */ + Pair getBackupStorageStats(Long zoneId); + + /** + * Gets the backup storage usage (Used, Total) from the plugin and stores it in db + * @param zoneId the zone for which to return metrics + */ + void syncBackupStorageStats(Long zoneId); + } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java b/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java index 8e5c9740e690..be539a0eb044 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java @@ -28,7 +28,9 @@ public interface BackupRepository extends InternalIdentity, Identity { String getType(); String getAddress(); String getMountOptions(); + void setUsedBytes(Long usedBytes); Long getCapacityBytes(); Long getUsedBytes(); + void setCapacityBytes(Long capacityBytes); Date getCreated(); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java b/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java index 26adc80db37a..b5138d34de11 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java @@ -30,6 +30,7 @@ public interface BackupSchedule extends InternalIdentity { String getTimezone(); Date getScheduledTimestamp(); Long getAsyncJobId(); + Boolean getQuiesceVM(); int getMaxBackups(); String getUuid(); } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java index a69a7a858ce0..c7aeb8ba99bf 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java @@ -61,6 +61,8 @@ public class AddObjectStoragePoolCmdTest { String provider = "Simulator"; + Long size = 10L; + Map details; private AutoCloseable closeable; @@ -74,6 +76,7 @@ public void setUp() throws Exception { ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "url", url); ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "providerName", provider); ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "details", details); + ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "size", size); addObjectStoragePoolCmdSpy._storageService = storageService; addObjectStoragePoolCmdSpy._responseGenerator = responseGenerator; } @@ -87,12 +90,12 @@ public void tearDown() throws Exception { @Test public void testAddObjectStore() throws DiscoveryException { Mockito.doReturn(objectStore).when(storageService).discoverObjectStore(Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), any()); + Mockito.anyString(), Mockito.anyLong(), Mockito.anyString(), any()); ObjectStoreResponse objectStoreResponse = new ObjectStoreResponse(); Mockito.doReturn(objectStoreResponse).when(responseGenerator).createObjectStoreResponse(any()); addObjectStoragePoolCmdSpy.execute(); Mockito.verify(storageService, Mockito.times(1)) - .discoverObjectStore(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + .discoverObjectStore(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java new file mode 100644 index 000000000000..f7e3e38d9c3f --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmdTest.java @@ -0,0 +1,483 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.vm; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiConstants.BootMode; +import org.apache.cloudstack.api.ApiConstants.BootType; +import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; +import org.apache.cloudstack.vm.lease.VMLeaseManager; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.network.NetworkService; +import com.cloud.utils.db.EntityManager; +import com.cloud.vm.VmDetailConstants; +import com.cloud.network.Network; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.offering.DiskOffering; +import com.cloud.network.Network.IpAddresses; +import com.cloud.vm.VmDiskInfo; + +@RunWith(MockitoJUnitRunner.class) +public class DeployVMCmdTest { + + @Spy + private DeployVMCmd cmd = new DeployVMCmd(); + + @Test + public void testGetBootType_ValidUEFI() { + ReflectionTestUtils.setField(cmd, "bootType", "UEFI"); + + BootType result = cmd.getBootType(); + + assertEquals(BootType.UEFI, result); + } + + @Test + public void testGetBootTypeValidBIOS() { + ReflectionTestUtils.setField(cmd, "bootType", "BIOS"); + + BootType result = cmd.getBootType(); + + assertEquals(BootType.BIOS, result); + } + + @Test + public void testGetBootTypeInvalidValue() { + ReflectionTestUtils.setField(cmd, "bootType", "INVALID"); + + InvalidParameterValueException thrownException = assertThrows(InvalidParameterValueException.class, () -> { + cmd.getBootType(); + }); + assertTrue(thrownException.getMessage().contains("Invalid bootType INVALID")); + } + + @Test + public void testGetBootTypeNullValue() { + ReflectionTestUtils.setField(cmd, "bootType", null); + + BootType result = cmd.getBootType(); + + assertNull(result); + } + + @Test + public void testGetBootModeValidSecure() { + ReflectionTestUtils.setField(cmd, "bootMode", "SECURE"); + ReflectionTestUtils.setField(cmd, "bootType", "UEFI"); + + BootMode result = cmd.getBootMode(); + + assertEquals(BootMode.SECURE, result); + } + + @Test + public void testGetBootModeValidLegacy() { + ReflectionTestUtils.setField(cmd, "bootMode", "LEGACY"); + ReflectionTestUtils.setField(cmd, "bootType", "UEFI"); + + BootMode result = cmd.getBootMode(); + + assertEquals(BootMode.LEGACY, result); + } + + @Test + public void testGetBootModeInvalidValue() { + ReflectionTestUtils.setField(cmd, "bootMode", "INVALID"); + ReflectionTestUtils.setField(cmd, "bootType", "UEFI"); + + InvalidParameterValueException thrownException = assertThrows(InvalidParameterValueException.class, () -> { + cmd.getBootMode(); + }); + assertTrue(thrownException.getMessage().contains("Invalid bootmode: INVALID specified for VM: null. Valid values are: [LEGACY, SECURE]")); + } + + @Test + public void testGetBootModeUEFIWithoutBootMode() { + ReflectionTestUtils.setField(cmd, "bootMode", null); + ReflectionTestUtils.setField(cmd, "bootType", "UEFI"); + + InvalidParameterValueException thrownException = assertThrows(InvalidParameterValueException.class, () -> { + cmd.getBootMode(); + }); + assertTrue(thrownException.getMessage().contains("bootmode must be specified for the VM with boot type: UEFI. Valid values are: [LEGACY, SECURE]")); + } + + @Test + public void testGetDetails() { + ReflectionTestUtils.setField(cmd, "bootType", "UEFI"); + ReflectionTestUtils.setField(cmd, "bootMode", "SECURE"); + ReflectionTestUtils.setField(cmd, "rootdisksize", 100L); + ReflectionTestUtils.setField(cmd, "ioDriverPolicy", "native"); + ReflectionTestUtils.setField(cmd, "iothreadsEnabled", true); + ReflectionTestUtils.setField(cmd, "nicMultiqueueNumber", null); + ReflectionTestUtils.setField(cmd, "nicPackedVirtQueues", null); + ReflectionTestUtils.setField(cmd, "details", new HashMap<>()); + + Map result = cmd.getDetails(); + + assertEquals("SECURE", result.get("UEFI")); + assertEquals("100", result.get(VmDetailConstants.ROOT_DISK_SIZE)); + assertEquals("native", result.get(VmDetailConstants.IO_POLICY)); + assertEquals("true", result.get(VmDetailConstants.IOTHREADS)); + } + + @Test + public void testGetLeaseExpiryActionValidStop() { + ReflectionTestUtils.setField(cmd, "leaseExpiryAction", "STOP"); + + VMLeaseManager.ExpiryAction result = cmd.getLeaseExpiryAction(); + + assertEquals(VMLeaseManager.ExpiryAction.STOP, result); + } + + @Test + public void testGetLeaseExpiryActionValidDestroy() { + ReflectionTestUtils.setField(cmd, "leaseExpiryAction", "DESTROY"); + + VMLeaseManager.ExpiryAction result = cmd.getLeaseExpiryAction(); + + assertEquals(VMLeaseManager.ExpiryAction.DESTROY, result); + } + + @Test + public void testGetLeaseExpiryActionInvalidValue() { + ReflectionTestUtils.setField(cmd, "leaseExpiryAction", "INVALID"); + + InvalidParameterValueException thrownException = assertThrows(InvalidParameterValueException.class, () -> { + cmd.getLeaseExpiryAction(); + }); + assertTrue(thrownException.getMessage().contains("Invalid value configured for leaseexpiryaction")); + } + + @Test + public void testGetLeaseExpiryActionNullValue() { + ReflectionTestUtils.setField(cmd, "leaseExpiryAction", null); + + VMLeaseManager.ExpiryAction result = cmd.getLeaseExpiryAction(); + + assertNull(result); + } + + @Test + public void testGetIoDriverPolicyValidThrottle() { + ReflectionTestUtils.setField(cmd, "ioDriverPolicy", "native"); + + IoDriverPolicy result = cmd.getIoDriverPolicy(); + + assertEquals(IoDriverPolicy.valueOf("NATIVE"), result); + } + + @Test + public void testGetIoDriverPolicyInvalidValue() { + ReflectionTestUtils.setField(cmd, "ioDriverPolicy", "INVALID"); + + InvalidParameterValueException thrownException = assertThrows(InvalidParameterValueException.class, () -> { + cmd.getIoDriverPolicy(); + }); + assertTrue(thrownException.getMessage().contains("Invalid io policy INVALID")); + } + + @Test + public void testGetNetworkIds() { + List networkIds = Arrays.asList(1L, 2L, 3L); + ReflectionTestUtils.setField(cmd, "networkIds", networkIds); + ReflectionTestUtils.setField(cmd, "vAppNetworks", null); + ReflectionTestUtils.setField(cmd, "ipToNetworkList", null); + + List result = cmd.getNetworkIds(); + + assertEquals(networkIds, result); + } + + @Test + public void testGetNetworkIdsVAppNetworks() { + Map vAppNetworks = new HashMap<>(); + vAppNetworks.put("network1", new HashMap()); + ReflectionTestUtils.setField(cmd, "vAppNetworks", vAppNetworks); + ReflectionTestUtils.setField(cmd, "networkIds", null); + ReflectionTestUtils.setField(cmd, "ipToNetworkList", null); + ReflectionTestUtils.setField(cmd, "ipAddress", null); + ReflectionTestUtils.setField(cmd, "ip6Address", null); + + List result = cmd.getNetworkIds(); + + assertTrue(result.isEmpty()); + } + + @Test + public void testGetNetworkIdsVAppNetworksAndNetworkIds() { + Map vAppNetworks = new HashMap<>(); + vAppNetworks.put("network1", new HashMap()); + ReflectionTestUtils.setField(cmd, "vAppNetworks", vAppNetworks); + ReflectionTestUtils.setField(cmd, "networkIds", Arrays.asList(1L, 2L)); + + InvalidParameterValueException thrownException = assertThrows(InvalidParameterValueException.class, () -> { + cmd.getNetworkIds(); + }); + assertTrue(thrownException.getMessage().contains("nicnetworklist can't be specified along with networkids")); + } + + @Test + public void testGetNetworkIdsIpToNetworkListAndNetworkIds() { + Map ipToNetworkList = new HashMap<>(); + ipToNetworkList.put("0", new HashMap()); + ReflectionTestUtils.setField(cmd, "ipToNetworkList", ipToNetworkList); + ReflectionTestUtils.setField(cmd, "networkIds", Arrays.asList(1L, 2L)); + + InvalidParameterValueException thrownException = assertThrows(InvalidParameterValueException.class, () -> { + cmd.getNetworkIds(); + }); + assertTrue(thrownException.getMessage().contains("ipToNetworkMap can't be specified along with networkIds or ipAddress")); + } + + @Test + public void testGetIpToNetworkMap_WithNetworkIds() { + ReflectionTestUtils.setField(cmd, "networkIds", Arrays.asList(1L, 2L)); + ReflectionTestUtils.setField(cmd, "ipToNetworkList", new HashMap<>()); + + InvalidParameterValueException thrownException = assertThrows(InvalidParameterValueException.class, () -> { + cmd.getIpToNetworkMap(); + }); + assertTrue(thrownException.getMessage().contains("NetworkIds and ipAddress can't be specified along with ipToNetworkMap parameter")); + } + + @Test + public void testGetIpToNetworkMap_WithIpAddress() { + ReflectionTestUtils.setField(cmd, "ipAddress", "192.168.1.1"); + ReflectionTestUtils.setField(cmd, "ipToNetworkList", new HashMap<>()); + + InvalidParameterValueException thrownException = assertThrows(InvalidParameterValueException.class, () -> { + cmd.getIpToNetworkMap(); + }); + assertTrue(thrownException.getMessage().contains("NetworkIds and ipAddress can't be specified along with ipToNetworkMap parameter")); + } + + @Test + public void testGetIpToNetworkMap_WithEmptyIpToNetworkList() { + ReflectionTestUtils.setField(cmd, "networkIds", null); + ReflectionTestUtils.setField(cmd, "ipAddress", null); + ReflectionTestUtils.setField(cmd, "ipToNetworkList", new HashMap<>()); + + Map result = cmd.getIpToNetworkMap(); + + assertNull(result); + } + + @Test + public void testGetIpToNetworkMap_WithNullIpToNetworkList() { + ReflectionTestUtils.setField(cmd, "networkIds", null); + ReflectionTestUtils.setField(cmd, "ipAddress", null); + ReflectionTestUtils.setField(cmd, "ipToNetworkList", null); + + Map result = cmd.getIpToNetworkMap(); + + assertNull(result); + } + + @Test + public void testGetDataDiskInfoList() { + Map dataDisksDetails = new HashMap<>(); + Map dataDisk = new HashMap<>(); + dataDisk.put(ApiConstants.DISK_OFFERING_ID, "offering-uuid"); + dataDisk.put(ApiConstants.DEVICE_ID, "0"); + dataDisk.put(ApiConstants.MIN_IOPS, "1000"); + dataDisk.put(ApiConstants.MAX_IOPS, "2000"); + dataDisksDetails.put("0", dataDisk); + + ReflectionTestUtils.setField(cmd, "dataDisksDetails", dataDisksDetails); + + EntityManager entityMgr = mock(EntityManager.class); + ReflectionTestUtils.setField(cmd, "_entityMgr", entityMgr); + DiskOffering diskOffering = mock(DiskOffering.class); + when(diskOffering.getDiskSize()).thenReturn(1024 * 1024 * 1024L); + when(diskOffering.isCustomizedIops()).thenReturn(true); + when(entityMgr.findByUuid(DiskOffering.class, "offering-uuid")).thenReturn(diskOffering); + + List result = cmd.getDataDiskInfoList(); + + assertNotNull(result); + assertEquals(1, result.size()); + assertEquals(diskOffering, result.get(0).getDiskOffering()); + assertEquals(1L, result.get(0).getSize().longValue()); + assertEquals(1000L, result.get(0).getMinIops().longValue()); + assertEquals(2000L, result.get(0).getMaxIops().longValue()); + } + + @Test + public void testGetIpAddressesFromIpMap() { + Map ipToNetworkList = new HashMap<>(); + Map ipMap = new HashMap<>(); + ipMap.put("ip", "192.168.1.100"); + ipMap.put("mac", "00:11:22:33:44:55"); + ipMap.put("networkid", "1"); + ipToNetworkList.put("0", ipMap); + + ReflectionTestUtils.setField(cmd, "ipToNetworkList", ipToNetworkList); + ReflectionTestUtils.setField(cmd, "networkIds", null); + ReflectionTestUtils.setField(cmd, "ipAddress", null); + + Network mockNetwork = mock(Network.class); + NetworkService networkServiceMock = mock(NetworkService.class); + ReflectionTestUtils.setField(cmd, "_networkService", networkServiceMock); + + Map result = cmd.getIpToNetworkMap(); + + assertNotNull(result); + assertTrue(result.containsKey(1L)); + assertEquals(result.get(1L).getIp4Address(), "192.168.1.100"); + assertEquals(result.get(1L).getMacAddress(), "00:11:22:33:44:55"); + } + + @Test + public void testGetIpAddressesFromIpMapInvalidMac() { + Map ipToNetworkList = new HashMap<>(); + Map ipMap = new HashMap<>(); + ipMap.put("ip", "192.168.1.100"); + ipMap.put("mac", "invalid-mac"); + ipMap.put("networkid", "1"); + ipToNetworkList.put("0", ipMap); + + ReflectionTestUtils.setField(cmd, "ipToNetworkList", ipToNetworkList); + ReflectionTestUtils.setField(cmd, "networkIds", null); + ReflectionTestUtils.setField(cmd, "ipAddress", null); + + Network mockNetwork = mock(Network.class); + NetworkService networkServiceMock = mock(NetworkService.class); + ReflectionTestUtils.setField(cmd, "_networkService", networkServiceMock); + + InvalidParameterValueException thrownException = assertThrows(InvalidParameterValueException.class, () -> { + cmd.getIpToNetworkMap(); + }); + assertTrue(thrownException.getMessage().contains("Mac address is not valid")); + } + + @Test + public void testGetDhcpOptionsMap() { + Map dhcpOptionsNetworkList = new HashMap<>(); + Map dhcpOptions = new HashMap<>(); + dhcpOptions.put("networkid", "network-1"); + dhcpOptions.put("dhcp:114", "url-value"); + dhcpOptions.put("dhcp:66", "www.test.com"); + dhcpOptionsNetworkList.put("0", dhcpOptions); + + ReflectionTestUtils.setField(cmd, "dhcpOptionsNetworkList", dhcpOptionsNetworkList); + + Map> result = cmd.getDhcpOptionsMap(); + + assertNotNull(result); + assertTrue(result.containsKey("network-1")); + Map networkOptions = result.get("network-1"); + assertEquals("url-value", networkOptions.get(114)); + assertEquals("www.test.com", networkOptions.get(66)); + } + + @Test + public void testGetDhcpOptionsMap_WithMissingNetworkId() { + Map dhcpOptionsNetworkList = new HashMap<>(); + Map dhcpOptions = new HashMap<>(); + dhcpOptions.put("dhcp:114", "url-value"); + dhcpOptionsNetworkList.put("0", dhcpOptions); + + ReflectionTestUtils.setField(cmd, "dhcpOptionsNetworkList", dhcpOptionsNetworkList); + + IllegalArgumentException thrownException = assertThrows(IllegalArgumentException.class, () -> { + cmd.getDhcpOptionsMap(); + }); + assertTrue(thrownException.getMessage().contains("No networkid specified when providing extra dhcp options")); + } + + @Test + public void testGetDataDiskTemplateToDiskOfferingMap() { + ReflectionTestUtils.setField(cmd, "diskOfferingId", null); + + Map dataDiskTemplateToDiskOfferingList = new HashMap<>(); + Map dataDiskTemplate = new HashMap<>(); + dataDiskTemplate.put("datadisktemplateid", "template-uuid"); + dataDiskTemplate.put("diskofferingid", "offering-uuid"); + dataDiskTemplateToDiskOfferingList.put("0", dataDiskTemplate); + + ReflectionTestUtils.setField(cmd, "dataDiskTemplateToDiskOfferingList", dataDiskTemplateToDiskOfferingList); + + VirtualMachineTemplate mockTemplate = mock(VirtualMachineTemplate.class); + when(mockTemplate.getId()).thenReturn(1L); + + DiskOffering mockOffering = mock(DiskOffering.class); + + EntityManager entityMgr = mock(EntityManager.class); + ReflectionTestUtils.setField(cmd, "_entityMgr", entityMgr); + when(entityMgr.findByUuid(VirtualMachineTemplate.class, "template-uuid")).thenReturn(mockTemplate); + when(entityMgr.findByUuid(DiskOffering.class, "offering-uuid")).thenReturn(mockOffering); + + Map result = cmd.getDataDiskTemplateToDiskOfferingMap(); + + assertNotNull(result); + assertEquals(mockOffering, result.get(1L)); + } + + @Test + public void testGetDataDiskTemplateToDiskOfferingMapWithDiskOfferingId() { + ReflectionTestUtils.setField(cmd, "diskOfferingId", 1L); + ReflectionTestUtils.setField(cmd, "dataDiskTemplateToDiskOfferingList", new HashMap<>()); + + InvalidParameterValueException thrownException = assertThrows(InvalidParameterValueException.class, () -> { + cmd.getDataDiskTemplateToDiskOfferingMap(); + }); + assertTrue(thrownException.getMessage().contains("diskofferingid parameter can't be specified along with datadisktemplatetodiskofferinglist parameter")); + } + + @Test + public void testGetDataDiskTemplateToDiskOfferingMapInvalidTemplateId() { + ReflectionTestUtils.setField(cmd, "diskOfferingId", null); + + Map dataDiskTemplateToDiskOfferingList = new HashMap<>(); + Map dataDiskTemplate = new HashMap<>(); + dataDiskTemplate.put("datadisktemplateid", "invalid-template"); + dataDiskTemplate.put("diskofferingid", "offering-uuid"); + dataDiskTemplateToDiskOfferingList.put("0", dataDiskTemplate); + + ReflectionTestUtils.setField(cmd, "dataDiskTemplateToDiskOfferingList", dataDiskTemplateToDiskOfferingList); + + EntityManager entityMgr = mock(EntityManager.class); + ReflectionTestUtils.setField(cmd, "_entityMgr", entityMgr); + when(entityMgr.findByUuid(VirtualMachineTemplate.class, "invalid-template")).thenReturn(null); + when(entityMgr.findById(VirtualMachineTemplate.class, "invalid-template")).thenReturn(null); + + InvalidParameterValueException thrownException = assertThrows(InvalidParameterValueException.class, () -> { + cmd.getDataDiskTemplateToDiskOfferingMap(); + }); + assertTrue(thrownException.getMessage().contains("Unable to translate and find entity with datadisktemplateid")); + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java index 09f9c5621502..ffc67b628a7e 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java +++ b/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java @@ -28,6 +28,7 @@ public class BackupAnswer extends Answer { private Long size; private Long virtualSize; private Map volumes; + Boolean needsCleanup; public BackupAnswer(final Command command, final boolean success, final String details) { super(command, success, details); @@ -56,4 +57,15 @@ public Map getVolumes() { public void setVolumes(Map volumes) { this.volumes = volumes; } + + public Boolean getNeedsCleanup() { + if (needsCleanup == null) { + return false; + } + return needsCleanup; + } + + public void setNeedsCleanup(Boolean needsCleanup) { + this.needsCleanup = needsCleanup; + } } diff --git a/core/src/main/java/org/apache/cloudstack/backup/BackupStorageStatsAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/BackupStorageStatsAnswer.java new file mode 100644 index 000000000000..eabf6877ba6e --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/BackupStorageStatsAnswer.java @@ -0,0 +1,50 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; + +public class BackupStorageStatsAnswer extends Answer { + private Long totalSize; + private Long usedSize; + + public BackupStorageStatsAnswer(final Command command, final boolean success, final String details) { + super(command, success, details); + this.totalSize = 0L; + this.usedSize = 0L; + } + + public Long getTotalSize() { + return totalSize; + } + + public void setTotalSize(Long totalSize) { + this.totalSize = totalSize; + } + + public Long getUsedSize() { + return usedSize; + } + + public void setUsedSize(Long usedSize) { + this.usedSize = usedSize; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/GetBackupStorageStatsCommand.java b/core/src/main/java/org/apache/cloudstack/backup/GetBackupStorageStatsCommand.java new file mode 100644 index 000000000000..1ceeac17e52e --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/GetBackupStorageStatsCommand.java @@ -0,0 +1,66 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; + +public class GetBackupStorageStatsCommand extends Command { + private String backupRepoType; + private String backupRepoAddress; + @LogLevel(LogLevel.Log4jLevel.Off) + private String mountOptions; + + public GetBackupStorageStatsCommand(String backupRepoType, String backupRepoAddress, String mountOptions) { + super(); + this.backupRepoType = backupRepoType; + this.backupRepoAddress = backupRepoAddress; + this.mountOptions = mountOptions; + } + + public String getBackupRepoType() { + return backupRepoType; + } + + public void setBackupRepoType(String backupRepoType) { + this.backupRepoType = backupRepoType; + } + + public String getBackupRepoAddress() { + return backupRepoAddress; + } + + public void setBackupRepoAddress(String backupRepoAddress) { + this.backupRepoAddress = backupRepoAddress; + } + + public String getMountOptions() { + return mountOptions == null ? "" : mountOptions; + } + + public void setMountOptions(String mountOptions) { + this.mountOptions = mountOptions; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java index 7228e35147af..f447fbe3d008 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java @@ -30,7 +30,8 @@ public class RestoreBackupCommand extends Command { private String backupPath; private String backupRepoType; private String backupRepoAddress; - private List volumePaths; + private List backupVolumesUUIDs; + private List restoreVolumePaths; private String diskType; private Boolean vmExists; private String restoreVolumeUUID; @@ -72,12 +73,12 @@ public void setBackupRepoAddress(String backupRepoAddress) { this.backupRepoAddress = backupRepoAddress; } - public List getVolumePaths() { - return volumePaths; + public List getRestoreVolumePaths() { + return restoreVolumePaths; } - public void setVolumePaths(List volumePaths) { - this.volumePaths = volumePaths; + public void setRestoreVolumePaths(List restoreVolumePaths) { + this.restoreVolumePaths = restoreVolumePaths; } public Boolean isVmExists() { @@ -127,4 +128,12 @@ public void setVmState(VirtualMachine.State vmState) { public boolean executeInSequence() { return true; } + + public List getBackupVolumesUUIDs() { + return backupVolumesUUIDs; + } + + public void setBackupVolumesUUIDs(List backupVolumesUUIDs) { + this.backupVolumesUUIDs = backupVolumesUUIDs; + } } diff --git a/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java index 93855ea17211..ecebd57a178c 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java @@ -30,6 +30,7 @@ public class TakeBackupCommand extends Command { private String backupRepoType; private String backupRepoAddress; private List volumePaths; + private Boolean quiesce; @LogLevel(LogLevel.Log4jLevel.Off) private String mountOptions; @@ -87,6 +88,14 @@ public void setVolumePaths(List volumePaths) { this.volumePaths = volumePaths; } + public Boolean getQuiesce() { + return quiesce; + } + + public void setQuiesce(Boolean quiesce) { + this.quiesce = quiesce; + } + @Override public boolean executeInSequence() { return true; diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index 7841eba524ac..c05c29add556 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -124,6 +124,7 @@ interface Topics { * @param defaultNetwork The default network for the VM. * @param rootDiskOffering For created VMs not based on templates, root disk offering specifies the root disk. * @param dataDiskOfferings Data disks to attach to the VM. + * @param dataDiskDeviceIds Device Ids to assign the data disks to. * @param auxiliaryNetworks additional networks to attach the VMs to. * @param plan How to deploy the VM. * @param hyperType Hypervisor type @@ -131,7 +132,7 @@ interface Topics { * @throws InsufficientCapacityException If there are insufficient capacity to deploy this vm. */ void allocate(String vmInstanceName, VirtualMachineTemplate template, ServiceOffering serviceOffering, DiskOfferingInfo rootDiskOfferingInfo, - List dataDiskOfferings, LinkedHashMap> auxiliaryNetworks, DeploymentPlan plan, + List dataDiskOfferings, List dataDiskDeviceIds, LinkedHashMap> auxiliaryNetworks, DeploymentPlan plan, HypervisorType hyperType, Map> extraDhcpOptions, Map datadiskTemplateToDiskOfferingMap, Volume volume, Snapshot snapshot) throws InsufficientCapacityException; void allocate(String vmInstanceName, VirtualMachineTemplate template, ServiceOffering serviceOffering, diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java index ffe85818fc46..6be71b3cb250 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java @@ -40,6 +40,7 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.offering.DiskOffering; import com.cloud.vm.NicProfile; +import com.cloud.vm.VmDiskInfo; @Path("orchestration") @Produces({"application/json", "application/xml"}) @@ -69,15 +70,17 @@ VirtualMachineEntity createVirtualMachine(@QueryParam("id") String id, @QueryPar @QueryParam("compute-tags") List computeTags, @QueryParam("root-disk-tags") List rootDiskTags, @QueryParam("network-nic-map") Map> networkNicMap, @QueryParam("deploymentplan") DeploymentPlan plan, @QueryParam("root-disk-size") Long rootDiskSize, @QueryParam("extra-dhcp-option-map") Map> extraDhcpOptionMap, - @QueryParam("datadisktemplate-diskoffering-map") Map datadiskTemplateToDiskOfferingMap, @QueryParam("disk-offering-id") Long diskOfferingId, @QueryParam("root-disk-offering-id") Long rootDiskOfferingId, Volume volume, Snapshot snapshot) throws InsufficientCapacityException; + @QueryParam("datadisktemplate-diskoffering-map") Map datadiskTemplateToDiskOfferingMap, @QueryParam("disk-offering-id") Long diskOfferingId, + @QueryParam("root-disk-offering-id") Long rootDiskOfferingId, List dataDiskInfoList, Volume volume, Snapshot snapshot) throws InsufficientCapacityException; @POST VirtualMachineEntity createVirtualMachineFromScratch(@QueryParam("id") String id, @QueryParam("owner") String owner, @QueryParam("iso-id") String isoId, - @QueryParam("host-name") String hostName, @QueryParam("display-name") String displayName, @QueryParam("hypervisor") String hypervisor, - @QueryParam("os") String os, @QueryParam("cpu") int cpu, @QueryParam("speed") int speed, @QueryParam("ram") long memory, @QueryParam("disk-size") Long diskSize, - @QueryParam("compute-tags") List computeTags, @QueryParam("root-disk-tags") List rootDiskTags, - @QueryParam("network-nic-map") Map> networkNicMap, @QueryParam("deploymentplan") DeploymentPlan plan, - @QueryParam("extra-dhcp-option-map") Map> extraDhcpOptionMap, @QueryParam("disk-offering-id") Long diskOfferingId, Volume volume, Snapshot snapshot) throws InsufficientCapacityException; + @QueryParam("host-name") String hostName, @QueryParam("display-name") String displayName, @QueryParam("hypervisor") String hypervisor, + @QueryParam("os") String os, @QueryParam("cpu") int cpu, @QueryParam("speed") int speed, @QueryParam("ram") long memory, @QueryParam("disk-size") Long diskSize, + @QueryParam("compute-tags") List computeTags, @QueryParam("root-disk-tags") List rootDiskTags, + @QueryParam("network-nic-map") Map> networkNicMap, @QueryParam("deploymentplan") DeploymentPlan plan, + @QueryParam("extra-dhcp-option-map") Map> extraDhcpOptionMap, @QueryParam("disk-offering-id") Long diskOfferingId, + @QueryParam("data-disks-offering-info") List dataDiskInfoList, Volume volume, Snapshot snapshot) throws InsufficientCapacityException; @POST NetworkEntity createNetwork(String id, String name, String domainName, String cidr, String gateway); diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 3fc6d80befee..d3a6d4525a5e 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -220,6 +220,14 @@ public interface StorageManager extends StorageService { "storage.pool.host.connect.workers", "1", "Number of worker threads to be used to connect hosts to a primary storage", true); + ConfigKey ObjectStorageCapacityThreshold = new ConfigKey<>("Alert", Float.class, + "objectStorage.capacity.notificationthreshold", + "0.75", + "Percentage (as a value between 0 and 1) of object storage utilization above which alerts will be sent about low storage available.", + true, + ConfigKey.Scope.Global, + null); + /** * should we execute in sequence not involving any storages? * @return tru if commands should execute in sequence @@ -415,4 +423,6 @@ void connectHostsToPool(DataStore primaryStore, List hostIds, Scope scope, Pair checkIfReadyVolumeFitsInStoragePoolWithStorageAccessGroups(StoragePool destPool, Volume volume); String[] getStorageAccessGroups(Long zoneId, Long podId, Long clusterId, Long hostId); + + CapacityVO getObjectStorageUsedStats(Long zoneId); } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 3cd8ec0aae35..3a6e1b622774 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -58,6 +58,8 @@ import org.apache.cloudstack.api.command.admin.vm.MigrateVMCmd; import org.apache.cloudstack.api.command.admin.volume.MigrateVolumeCmdByAdmin; import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; @@ -437,6 +439,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac VmWorkJobDao vmWorkJobDao; @Inject DataStoreProviderManager dataStoreProviderManager; + @Inject + BackupManager backupManager; + @Inject + BackupDao backupDao; private SingleCache> vmIdsInProgressCache; @@ -525,8 +531,9 @@ public void registerGuru(final VirtualMachine.Type type, final VirtualMachineGur @Override @DB public void allocate(final String vmInstanceName, final VirtualMachineTemplate template, final ServiceOffering serviceOffering, - final DiskOfferingInfo rootDiskOfferingInfo, final List dataDiskOfferings, - final LinkedHashMap> auxiliaryNetworks, final DeploymentPlan plan, final HypervisorType hyperType, final Map> extraDhcpOptions, final Map datadiskTemplateToDiskOfferingMap, Volume volume, Snapshot snapshot) + final DiskOfferingInfo rootDiskOfferingInfo, final List dataDiskOfferings, List dataDiskDeviceIds, + final LinkedHashMap> auxiliaryNetworks,final DeploymentPlan plan, final HypervisorType hyperType, + final Map> extraDhcpOptions, final Map datadiskTemplateToDiskOfferingMap, Volume volume, Snapshot snapshot) throws InsufficientCapacityException { logger.info("allocating virtual machine from template: {} with hostname: {} and {} networks", template, vmInstanceName, auxiliaryNetworks.size()); @@ -570,19 +577,22 @@ public void allocate(final String vmInstanceName, final VirtualMachineTemplate t CallContext volumeContext = CallContext.register(CallContext.current(), ApiCommandResourceType.Volume); try { if (dataDiskOfferings != null) { + int index = 0; for (final DiskOfferingInfo dataDiskOfferingInfo : dataDiskOfferings) { - volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId(), dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(), - dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), persistedVm, template, owner, null); + Long deviceId = dataDiskDeviceIds.get(index++); + String volumeName = deviceId == null ? "DATA-" + persistedVm.getId() : "DATA-" + persistedVm.getId() + "-" + String.valueOf(deviceId); + volumeMgr.allocateRawVolume(Type.DATADISK, volumeName, dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(), + dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), persistedVm, template, owner, deviceId); } } if (datadiskTemplateToDiskOfferingMap != null && !datadiskTemplateToDiskOfferingMap.isEmpty()) { - int diskNumber = 1; + Long diskNumber = 1L; for (Entry dataDiskTemplateToDiskOfferingMap : datadiskTemplateToDiskOfferingMap.entrySet()) { DiskOffering diskOffering = dataDiskTemplateToDiskOfferingMap.getValue(); long diskOfferingSize = diskOffering.getDiskSize() / (1024 * 1024 * 1024); VMTemplateVO dataDiskTemplate = _templateDao.findById(dataDiskTemplateToDiskOfferingMap.getKey()); - volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId() + "-" + String.valueOf(diskNumber), diskOffering, diskOfferingSize, null, null, - persistedVm, dataDiskTemplate, owner, Long.valueOf(diskNumber)); + volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId() + "-" + String.valueOf( diskNumber), diskOffering, diskOfferingSize, null, null, + persistedVm, dataDiskTemplate, owner, diskNumber); diskNumber++; } } @@ -629,7 +639,7 @@ private void allocateRootVolume(VMInstanceVO vm, VirtualMachineTemplate template public void allocate(final String vmInstanceName, final VirtualMachineTemplate template, final ServiceOffering serviceOffering, final LinkedHashMap> networks, final DeploymentPlan plan, final HypervisorType hyperType, Volume volume, Snapshot snapshot) throws InsufficientCapacityException { DiskOffering diskOffering = _diskOfferingDao.findById(serviceOffering.getDiskOfferingId()); - allocate(vmInstanceName, template, serviceOffering, new DiskOfferingInfo(diskOffering), new ArrayList<>(), networks, plan, hyperType, null, null, volume, snapshot); + allocate(vmInstanceName, template, serviceOffering, new DiskOfferingInfo(diskOffering), new ArrayList<>(), new ArrayList<>(), networks, plan, hyperType, null, null, volume, snapshot); } VirtualMachineGuru getVmGuru(final VirtualMachine vm) { @@ -2589,6 +2599,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) throws throw new CloudRuntimeException("Unable to destroy " + vm); } else { if (expunge) { + backupManager.checkAndRemoveBackupOfferingBeforeExpunge(vm); if (!stateTransitTo(vm, VirtualMachine.Event.ExpungeOperation, vm.getHostId())) { logger.debug("Unable to expunge the vm because it is not in the correct state: {}", vm); throw new CloudRuntimeException("Unable to expunge " + vm); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java index c9af76fbdddf..8639f006383f 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java @@ -60,6 +60,7 @@ import com.cloud.vm.NicProfile; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.VmDiskInfo; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.dao.VMInstanceDao; @@ -162,7 +163,8 @@ public void destroyVolume(String volumeEntity) { @Override public VirtualMachineEntity createVirtualMachine(String id, String owner, String templateId, String hostName, String displayName, String hypervisor, int cpu, int speed, long memory, Long diskSize, List computeTags, List rootDiskTags, Map> networkNicMap, DeploymentPlan plan, - Long rootDiskSize, Map> extraDhcpOptionMap, Map dataDiskTemplateToDiskOfferingMap, Long dataDiskOfferingId, Long rootDiskOfferingId, Volume volume, Snapshot snapshot) throws InsufficientCapacityException { + Long rootDiskSize, Map> extraDhcpOptionMap, Map dataDiskTemplateToDiskOfferingMap, Long dataDiskOfferingId, Long rootDiskOfferingId, + List dataDiskInfoList, Volume volume, Snapshot snapshot) throws InsufficientCapacityException { // VirtualMachineEntityImpl vmEntity = new VirtualMachineEntityImpl(id, owner, hostName, displayName, cpu, speed, memory, computeTags, rootDiskTags, networks, // vmEntityManager); @@ -187,7 +189,6 @@ public VirtualMachineEntity createVirtualMachine(String id, String owner, String // Else, a disk offering is optional, and if present will be used to create the data disk DiskOfferingInfo rootDiskOfferingInfo = new DiskOfferingInfo(); - List dataDiskOfferings = new ArrayList(); ServiceOfferingVO computeOffering = _serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId()); @@ -210,6 +211,8 @@ public VirtualMachineEntity createVirtualMachine(String id, String owner, String } } + List dataDiskOfferings = new ArrayList(); + List dataDiskDeviceIds = new ArrayList<>(); if (dataDiskOfferingId != null) { DiskOfferingVO diskOffering = _diskOfferingDao.findById(dataDiskOfferingId); if (diskOffering == null) { @@ -243,6 +246,12 @@ public VirtualMachineEntity createVirtualMachine(String id, String owner, String } dataDiskOfferings.add(dataDiskOfferingInfo); + dataDiskDeviceIds.add(null); + } + } else if (dataDiskInfoList != null){ + dataDiskOfferings.addAll(dataDiskInfoList); + for (VmDiskInfo dataDiskInfo : dataDiskInfoList) { + dataDiskDeviceIds.add(dataDiskInfo.getDeviceId()); } } @@ -262,8 +271,8 @@ public VirtualMachineEntity createVirtualMachine(String id, String owner, String template = _templateDao.findByIdIncludingRemoved(new Long(templateId)); } else template = _templateDao.findById(new Long(templateId)); - _itMgr.allocate(vm.getInstanceName(), template, computeOffering, rootDiskOfferingInfo, dataDiskOfferings, networkIpMap, plan, - hypervisorType, extraDhcpOptionMap, dataDiskTemplateToDiskOfferingMap, volume, snapshot); + _itMgr.allocate(vm.getInstanceName(), template, computeOffering, rootDiskOfferingInfo, dataDiskOfferings, dataDiskDeviceIds, + networkIpMap, plan, hypervisorType, extraDhcpOptionMap, dataDiskTemplateToDiskOfferingMap, volume, snapshot); return vmEntity; } @@ -271,7 +280,7 @@ public VirtualMachineEntity createVirtualMachine(String id, String owner, String @Override public VirtualMachineEntity createVirtualMachineFromScratch(String id, String owner, String isoId, String hostName, String displayName, String hypervisor, String os, int cpu, int speed, long memory, Long diskSize, List computeTags, List rootDiskTags, Map> networkNicMap, DeploymentPlan plan, - Map> extraDhcpOptionMap, Long diskOfferingId, Volume volume, Snapshot snapshot) + Map> extraDhcpOptionMap, Long diskOfferingId, List dataDiskInfoList, Volume volume, Snapshot snapshot) throws InsufficientCapacityException { // VirtualMachineEntityImpl vmEntity = new VirtualMachineEntityImpl(id, owner, hostName, displayName, cpu, speed, memory, computeTags, rootDiskTags, networks, vmEntityManager); @@ -317,6 +326,14 @@ public VirtualMachineEntity createVirtualMachineFromScratch(String id, String ow rootDiskOfferingInfo.setMaxIops(maxIops != null && maxIops.trim().length() > 0 ? Long.parseLong(maxIops) : null); } } + List dataDiskOfferings = new ArrayList<>(); + List dataDiskDeviceIds = new ArrayList<>(); + if (dataDiskInfoList != null) { + dataDiskOfferings.addAll(dataDiskInfoList); + for (VmDiskInfo dataDiskInfo : dataDiskInfoList) { + dataDiskDeviceIds.add(dataDiskInfo.getDeviceId()); + } + } LinkedHashMap> networkIpMap = new LinkedHashMap>(); for (String uuid : networkNicMap.keySet()) { @@ -328,7 +345,8 @@ public VirtualMachineEntity createVirtualMachineFromScratch(String id, String ow HypervisorType hypervisorType = HypervisorType.valueOf(hypervisor); - _itMgr.allocate(vm.getInstanceName(), _templateDao.findByIdIncludingRemoved(new Long(isoId)), computeOffering, rootDiskOfferingInfo, new ArrayList(), networkIpMap, plan, hypervisorType, extraDhcpOptionMap, null, volume, snapshot); + _itMgr.allocate(vm.getInstanceName(), _templateDao.findByIdIncludingRemoved(new Long(isoId)), computeOffering, rootDiskOfferingInfo, dataDiskOfferings, dataDiskDeviceIds, + networkIpMap, plan, hypervisorType, extraDhcpOptionMap, null, volume, snapshot); return vmEntity; } diff --git a/engine/schema/src/main/java/com/cloud/capacity/CapacityVO.java b/engine/schema/src/main/java/com/cloud/capacity/CapacityVO.java index cd62935f17ee..fb2d61d8e11a 100644 --- a/engine/schema/src/main/java/com/cloud/capacity/CapacityVO.java +++ b/engine/schema/src/main/java/com/cloud/capacity/CapacityVO.java @@ -254,6 +254,8 @@ public String getUuid() { capacityNames.put(CAPACITY_TYPE_GPU, "GPU"); capacityNames.put(CAPACITY_TYPE_CPU_CORE, "CPU_CORE"); capacityNames.put(CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET, "VIRTUAL_NETWORK_IPV6_SUBNET"); + capacityNames.put(CAPACITY_TYPE_BACKUP_STORAGE, "BACKUP_STORAGE"); + capacityNames.put(CAPACITY_TYPE_OBJECT_STORAGE, "OBJECT_STORAGE"); } public static String getCapacityName (Short capacityType) { diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageBackupDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageBackupDao.java index 8a72182ec677..2a24016653db 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageBackupDao.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageBackupDao.java @@ -24,7 +24,7 @@ import com.cloud.utils.db.GenericDao; public interface UsageBackupDao extends GenericDao { - void updateMetrics(Long vmId, Long size, Long virtualSize); - void removeUsage(Long accountId, Long vmId, Date eventDate); + void updateMetrics(Long vmId, Long backupOfferingId, Long size, Long virtualSize); + void removeUsage(Long accountId, Long vmId, Long backupOfferingId, Date eventDate); List getUsageRecords(Long accountId, Date startDate, Date endDate); } diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageBackupDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageBackupDaoImpl.java index 3403a8dfe5bb..e5b46b02a591 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageBackupDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageBackupDaoImpl.java @@ -36,16 +36,17 @@ @Component public class UsageBackupDaoImpl extends GenericDaoBase implements UsageBackupDao { - protected static final String UPDATE_DELETED = "UPDATE usage_backup SET removed = ? WHERE account_id = ? AND vm_id = ? and removed IS NULL"; + protected static final String UPDATE_DELETED = "UPDATE usage_backup SET removed = ? WHERE account_id = ? AND vm_id = ? and backup_offering_id = ? and removed IS NULL"; protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT id, zone_id, account_id, domain_id, vm_id, backup_offering_id, size, protected_size, created, removed FROM usage_backup WHERE " + " account_id = ? AND ((removed IS NULL AND created <= ?) OR (created BETWEEN ? AND ?) OR (removed BETWEEN ? AND ?) " + " OR ((created <= ?) AND (removed >= ?)))"; @Override - public void updateMetrics(final Long vmId, final Long size, final Long virtualSize) { + public void updateMetrics(final Long vmId, Long backupOfferingId, final Long size, final Long virtualSize) { try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB)) { SearchCriteria sc = this.createSearchCriteria(); sc.addAnd("vmId", SearchCriteria.Op.EQ, vmId); + sc.addAnd("backupOfferingId", SearchCriteria.Op.EQ, backupOfferingId); UsageBackupVO vo = findOneBy(sc); if (vo != null) { vo.setSize(size); @@ -58,7 +59,7 @@ public void updateMetrics(final Long vmId, final Long size, final Long virtualSi } @Override - public void removeUsage(Long accountId, Long vmId, Date eventDate) { + public void removeUsage(Long accountId, Long vmId, Long backupOfferingId, Date eventDate) { TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); try { txn.start(); @@ -67,6 +68,7 @@ public void removeUsage(Long accountId, Long vmId, Date eventDate) { pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), eventDate)); pstmt.setLong(2, accountId); pstmt.setLong(3, vmId); + pstmt.setLong(3, backupOfferingId); pstmt.executeUpdate(); } } catch (SQLException e) { diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 823642d8c3d7..f722b4c54e4a 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -118,7 +118,7 @@ public interface VMInstanceDao extends GenericDao, StateDao< List listVmsMigratingFromHost(Long hostId); - List listByZoneWithBackups(Long zoneId, Long backupOfferingId); + List listByZoneAndBackupOffering(Long zoneId, Long backupOfferingId); public Long countActiveByHostId(long hostId); @@ -187,4 +187,5 @@ List searchRemovedByRemoveDate(final Date startDate, final Date en Map getNameIdMapForVmIds(Collection ids); + List listByIdsIncludingRemoved(List ids); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index dc0391f71fd9..2f19d36c37a2 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -661,7 +661,7 @@ public List listVmsMigratingFromHost(Long hostId) { } @Override - public List listByZoneWithBackups(Long zoneId, Long backupOfferingId) { + public List listByZoneAndBackupOffering(Long zoneId, Long backupOfferingId) { SearchCriteria sc = BackupSearch.create(); sc.setParameters("zone_id", zoneId); if (backupOfferingId != null) { @@ -1246,4 +1246,14 @@ public Map getNameIdMapForVmIds(Collection ids) { return vms.stream() .collect(Collectors.toMap(VMInstanceVO::getInstanceName, VMInstanceVO::getId)); } + + @Override + public List listByIdsIncludingRemoved(List ids) { + SearchBuilder idsSearch = createSearchBuilder(); + idsSearch.and("ids", idsSearch.entity().getId(), SearchCriteria.Op.IN); + idsSearch.done(); + SearchCriteria sc = idsSearch.create(); + sc.setParameters("ids", ids.toArray()); + return listIncludingRemovedBy(sc); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupDetailVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupDetailVO.java new file mode 100644 index 000000000000..aaf63518708c --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupDetailVO.java @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.apache.cloudstack.api.ResourceDetail; + +@Entity +@Table(name = "backup_details") +public class BackupDetailVO implements ResourceDetail { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "backup_id") + private long resourceId; + + @Column(name = "name") + private String name; + + @Column(name = "value", length = 65536) + private String value; + + @Column(name = "display") + private boolean display = true; + + public BackupDetailVO() { + } + + public BackupDetailVO(long backupId, String name, String value, boolean display) { + this.resourceId = backupId; + this.name = name; + this.value = value; + this.display = display; + } + + @Override + public long getId() { + return id; + } + + @Override + public long getResourceId() { + return resourceId; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getValue() { + return value; + } + + @Override + public boolean isDisplay() { + return display; + } + + public void setId(long id) { + this.id = id; + } + + public void setResourceId(long resourceId) { + this.resourceId = resourceId; + } + + public void setName(String name) { + this.name = name; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java index e8364520ed05..98efa94ceca2 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java @@ -144,11 +144,21 @@ public Long getUsedBytes() { return usedBytes; } + @Override + public void setUsedBytes(Long usedBytes) { + this.usedBytes = usedBytes; + } + @Override public Long getCapacityBytes() { return capacityBytes; } + @Override + public void setCapacityBytes(Long capacityBytes) { + this.capacityBytes = capacityBytes; + } + public Date getCreated() { return created; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java index 06e1dcfb1ed6..37e8105e3d51 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java @@ -65,16 +65,20 @@ public class BackupScheduleVO implements BackupSchedule { @Column(name = "max_backups") private int maxBackups = 0; + @Column(name = "quiescevm") + Boolean quiesceVM = false; + public BackupScheduleVO() { } - public BackupScheduleVO(Long vmId, DateUtil.IntervalType scheduleType, String schedule, String timezone, Date scheduledTimestamp, int maxBackups) { + public BackupScheduleVO(Long vmId, DateUtil.IntervalType scheduleType, String schedule, String timezone, Date scheduledTimestamp, int maxBackups, Boolean quiesceVM) { this.vmId = vmId; this.scheduleType = (short) scheduleType.ordinal(); this.schedule = schedule; this.timezone = timezone; this.scheduledTimestamp = scheduledTimestamp; this.maxBackups = maxBackups; + this.quiesceVM = quiesceVM; } @Override @@ -149,4 +153,12 @@ public int getMaxBackups() { public void setMaxBackups(int maxBackups) { this.maxBackups = maxBackups; } + + public void setQuiesceVM(Boolean quiesceVM) { + this.quiesceVM = quiesceVM; + } + + public Boolean getQuiesceVM() { + return quiesceVM; + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index 40bf97121376..0f8a10fb7be6 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -19,6 +19,7 @@ import com.cloud.utils.db.GenericDao; import com.google.gson.Gson; + import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.lang3.StringUtils; @@ -26,6 +27,7 @@ import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.UUID; import javax.persistence.Column; @@ -38,6 +40,7 @@ import javax.persistence.Table; import javax.persistence.Temporal; import javax.persistence.TemporalType; +import javax.persistence.Transient; @Entity @Table(name = "backups") @@ -47,11 +50,17 @@ public class BackupVO implements Backup { @Column(name = "id") private long id; + @Column(name = "name") + private String name; + + @Column(name = "description") + private String description; + @Column(name = "uuid") private String uuid; @Column(name = "vm_id") - private long vmId; + private Long vmId; @Column(name = "external_id") private String externalId; @@ -94,6 +103,9 @@ public class BackupVO implements Backup { @Column(name = "backup_schedule_id") private Long backupScheduleId; + @Transient + Map details; + public BackupVO() { this.uuid = UUID.randomUUID().toString(); } @@ -115,11 +127,11 @@ public String getUuid() { } @Override - public long getVmId() { + public Long getVmId() { return vmId; } - public void setVmId(long vmId) { + public void setVmId(Long vmId) { this.vmId = vmId; } @@ -218,7 +230,22 @@ public Class getEntityType() { @Override public String getName() { - return null; + return name; + } + + @Override + public void setName(String name) { + this.name = name; + } + + @Override + public String getDescription() { + return description; + } + + @Override + public void setDescription(String description) { + this.description = description; } public List getBackedUpVolumes() { @@ -232,10 +259,23 @@ public void setBackedUpVolumes(String backedUpVolumes) { this.backedUpVolumes = backedUpVolumes; } + @Override + public Map getDetails() { + return details; + } + + @Override + public String getDetail(String name) { + return this.details.get(name); + } + + public void setDetails(Map details) { + this.details = details; + } + public Date getRemoved() { return removed; } - public void setRemoved(Date removed) { this.removed = removed; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java index 64b8d5da5a90..e60e49e1a0c2 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java @@ -19,7 +19,6 @@ import java.util.List; -import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupVO; @@ -33,11 +32,14 @@ public interface BackupDao extends GenericDao { List listByVmId(Long zoneId, Long vmId); List listByAccountId(Long accountId); List syncBackups(Long zoneId, Long vmId, List externalBackups); + List listByVmIdAndOffering(Long zoneId, Long vmId, Long offeringId); + List searchByVmIds(List vmIds); BackupVO getBackupVO(Backup backup); List listByOfferingId(Long backupOfferingId); - - BackupResponse newBackupResponse(Backup backup); + List listVmIdsWithBackupsInZone(Long zoneId); public Long countBackupsForAccount(long accountId); public Long calculateBackupStorageForAccount(long accountId); + void loadDetails(BackupVO backup); + void saveDetails(BackupVO backup); List listBySchedule(Long backupScheduleId); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java index 0e8d8242f9cf..fd29da72c718 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java @@ -19,30 +19,31 @@ import java.util.ArrayList; import java.util.List; -import java.util.Objects; +import java.util.Map; import javax.annotation.PostConstruct; import javax.inject.Inject; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericSearchBuilder; -import org.apache.cloudstack.api.response.BackupResponse; + import org.apache.cloudstack.backup.Backup; -import org.apache.cloudstack.backup.BackupOffering; +import org.apache.cloudstack.backup.BackupDetailVO; import org.apache.cloudstack.backup.BackupVO; +import org.apache.commons.collections.CollectionUtils; -import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; -import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; -import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; -import com.cloud.vm.VMInstanceVO; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; import com.cloud.vm.dao.VMInstanceDao; -import com.google.gson.Gson; +import com.cloud.network.dao.NetworkDao; public class BackupDaoImpl extends GenericDaoBase implements BackupDao { @@ -58,13 +59,26 @@ public class BackupDaoImpl extends GenericDaoBase implements Bac @Inject VMInstanceDao vmInstanceDao; + @Inject + private VMTemplateDao templateDao; + @Inject BackupOfferingDao backupOfferingDao; + @Inject + BackupDetailsDao backupDetailsDao; + + @Inject + ServiceOfferingDao serviceOfferingDao; + + @Inject + NetworkDao networkDao; + private SearchBuilder backupSearch; private GenericSearchBuilder CountBackupsByAccount; private GenericSearchBuilder CalculateBackupStorageByAccount; private SearchBuilder listBackupsBySchedule; + private GenericSearchBuilder backupVmSearchInZone; public BackupDaoImpl() { } @@ -78,6 +92,11 @@ protected void init() { backupSearch.and("zone_id", backupSearch.entity().getZoneId(), SearchCriteria.Op.EQ); backupSearch.done(); + backupVmSearchInZone = createSearchBuilder(Long.class); + backupVmSearchInZone.select(null, SearchCriteria.Func.DISTINCT, backupVmSearchInZone.entity().getVmId()); + backupVmSearchInZone.and("zone_id", backupVmSearchInZone.entity().getZoneId(), SearchCriteria.Op.EQ); + backupVmSearchInZone.done(); + CountBackupsByAccount = createSearchBuilder(Long.class); CountBackupsByAccount.select(null, SearchCriteria.Func.COUNT, null); CountBackupsByAccount.and("account", CountBackupsByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); @@ -130,6 +149,17 @@ public List listByVmId(Long zoneId, Long vmId) { return new ArrayList<>(listBy(sc)); } + @Override + public List listByVmIdAndOffering(Long zoneId, Long vmId, Long offeringId) { + SearchCriteria sc = backupSearch.create(); + sc.setParameters("vm_id", vmId); + if (zoneId != null) { + sc.setParameters("zone_id", zoneId); + } + sc.setParameters("backup_offering_id", offeringId); + return new ArrayList<>(listBy(sc)); + } + private Backup findByExternalId(Long zoneId, String externalId) { SearchCriteria sc = backupSearch.create(); sc.setParameters("external_id", externalId); @@ -137,6 +167,18 @@ private Backup findByExternalId(Long zoneId, String externalId) { return findOneBy(sc); } + @Override + public List searchByVmIds(List vmIds) { + if (CollectionUtils.isEmpty(vmIds)) { + return new ArrayList<>(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("vmIds", sb.entity().getVmId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("vmIds", vmIds.toArray()); + return search(sc, null); + } + public BackupVO getBackupVO(Backup backup) { BackupVO backupVO = new BackupVO(); backupVO.setExternalId(backup.getExternalId()); @@ -158,6 +200,27 @@ public void removeExistingBackups(Long zoneId, Long vmId) { expunge(sc); } + @Override + public BackupVO persist(BackupVO backup) { + return Transaction.execute((TransactionCallback) status -> { + BackupVO backupDb = super.persist(backup); + saveDetails(backup); + loadDetails(backupDb); + return backupDb; + }); + } + + @Override + public boolean update(Long id, BackupVO backup) { + return Transaction.execute((TransactionCallback) status -> { + boolean result = super.update(id, backup); + if (result) { + saveDetails(backup); + } + return result; + }); + } + @Override public List syncBackups(Long zoneId, Long vmId, List externalBackups) { for (Backup backup : externalBackups) { @@ -171,7 +234,7 @@ public List syncBackups(Long zoneId, Long vmId, List externalBac public Long countBackupsForAccount(long accountId) { SearchCriteria sc = CountBackupsByAccount.create(); sc.setParameters("account", accountId); - sc.setParameters("status", Backup.Status.Error, Backup.Status.Failed, Backup.Status.Removed, Backup.Status.Expunged); + sc.setParameters("status", Backup.Status.Failed, Backup.Status.Removed, Backup.Status.Expunged); return customSearch(sc, null).get(0); } @@ -179,7 +242,7 @@ public Long countBackupsForAccount(long accountId) { public Long calculateBackupStorageForAccount(long accountId) { SearchCriteria sc = CalculateBackupStorageByAccount.create(); sc.setParameters("account", accountId); - sc.setParameters("status", Backup.Status.Error, Backup.Status.Failed, Backup.Status.Removed, Backup.Status.Expunged); + sc.setParameters("status", Backup.Status.Failed, Backup.Status.Removed, Backup.Status.Expunged); return customSearch(sc, null).get(0).sum; } @@ -192,44 +255,29 @@ public List listBySchedule(Long backupScheduleId) { } @Override - public BackupResponse newBackupResponse(Backup backup) { - VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); - AccountVO account = accountDao.findByIdIncludingRemoved(vm.getAccountId()); - DomainVO domain = domainDao.findByIdIncludingRemoved(vm.getDomainId()); - DataCenterVO zone = dataCenterDao.findByIdIncludingRemoved(vm.getDataCenterId()); - Long offeringId = backup.getBackupOfferingId(); - if (offeringId == null) { - offeringId = vm.getBackupOfferingId(); + public void loadDetails(BackupVO backup) { + Map details = backupDetailsDao.listDetailsKeyPairs(backup.getId()); + backup.setDetails(details); + } + + @Override + public void saveDetails(BackupVO backup) { + Map detailsStr = backup.getDetails(); + if (detailsStr == null) { + return; } - BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(offeringId); - - BackupResponse response = new BackupResponse(); - response.setId(backup.getUuid()); - response.setVmId(vm.getUuid()); - response.setVmName(vm.getHostName()); - response.setExternalId(backup.getExternalId()); - response.setType(backup.getType()); - response.setDate(backup.getDate()); - response.setSize(backup.getSize()); - response.setProtectedSize(backup.getProtectedSize()); - response.setStatus(backup.getStatus()); - // ACS 4.20: For backups taken prior this release the backup.backed_volumes column would be empty hence use vm_instance.backup_volumes - String backedUpVolumes; - if (Objects.isNull(backup.getBackedUpVolumes())) { - backedUpVolumes = new Gson().toJson(vm.getBackupVolumeList().toArray(), Backup.VolumeInfo[].class); - } else { - backedUpVolumes = new Gson().toJson(backup.getBackedUpVolumes().toArray(), Backup.VolumeInfo[].class); + List details = new ArrayList(); + for (String key : detailsStr.keySet()) { + BackupDetailVO detail = new BackupDetailVO(backup.getId(), key, detailsStr.get(key), true); + details.add(detail); } - response.setVolumes(backedUpVolumes); - response.setBackupOfferingId(offering.getUuid()); - response.setBackupOffering(offering.getName()); - response.setAccountId(account.getUuid()); - response.setAccount(account.getAccountName()); - response.setDomainId(domain.getUuid()); - response.setDomain(domain.getName()); - response.setZoneId(zone.getUuid()); - response.setZone(zone.getName()); - response.setObjectName("backup"); - return response; + backupDetailsDao.saveDetails(details); + } + + @Override + public List listVmIdsWithBackupsInZone(Long zoneId) { + SearchCriteria sc = backupVmSearchInZone.create(); + sc.setParameters("zone_id", zoneId); + return customSearchIncludingRemoved(sc, null); } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDetailsDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDetailsDao.java new file mode 100644 index 000000000000..664650074bce --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDetailsDao.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup.dao; + +import org.apache.cloudstack.backup.BackupDetailVO; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; + +import com.cloud.utils.db.GenericDao; + +public interface BackupDetailsDao extends GenericDao, ResourceDetailsDao { + +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDetailsDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDetailsDaoImpl.java new file mode 100644 index 000000000000..08c7192af909 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDetailsDaoImpl.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup.dao; + + +import org.apache.cloudstack.backup.BackupDetailVO; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import org.springframework.stereotype.Component; + +@Component +public class BackupDetailsDaoImpl extends ResourceDetailsDaoBase implements BackupDetailsDao { + + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new BackupDetailVO(resourceId, key, value, display)); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupOfferingDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupOfferingDaoImpl.java index 0568a0185bba..9d67d07fe5e3 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupOfferingDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupOfferingDaoImpl.java @@ -58,6 +58,7 @@ public BackupOfferingResponse newBackupOfferingResponse(BackupOffering offering) response.setName(offering.getName()); response.setDescription(offering.getDescription()); response.setExternalId(offering.getExternalId()); + response.setProvider(offering.getProvider()); response.setUserDrivenBackups(offering.isUserDrivenBackupAllowed()); if (zone != null) { response.setZoneId(zone.getUuid()); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java index 0034bfb30ab6..6dec994be0c9 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java @@ -28,4 +28,6 @@ public interface BackupRepositoryDao extends GenericDao listByZoneAndProvider(Long zoneId, String provider); BackupRepository findByBackupOfferingId(Long backupOfferingId); + + boolean updateCapacity(BackupRepository backupRepository, Long capacityBytes, Long usedBytes); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java index 460b6d8aba45..ea969988e2bb 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java @@ -64,4 +64,12 @@ public BackupRepository findByBackupOfferingId(Long backupOfferingId) { } return findByUuid(offering.getExternalId()); } + + @Override + public boolean updateCapacity(BackupRepository backupRepository, Long capacityBytes, Long usedBytes) { + BackupRepositoryVO repository = findById(backupRepository.getId()); + repository.setCapacityBytes(capacityBytes); + repository.setUsedBytes(usedBytes); + return update(repository.getId(), repository); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java index 9374798dde32..d9cf7b63680b 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java @@ -99,6 +99,9 @@ public BackupScheduleResponse newBackupScheduleResponse(BackupSchedule schedule) response.setSchedule(schedule.getSchedule()); response.setTimezone(schedule.getTimezone()); response.setMaxBackups(schedule.getMaxBackups()); + if (schedule.getQuiesceVM() != null) { + response.setQuiesceVM(schedule.getQuiesceVM()); + } response.setObjectName("backupschedule"); return response; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDao.java index 94f6b5ec3724..695742823ebe 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDao.java @@ -39,4 +39,6 @@ public interface ObjectStoreDao extends GenericDao { ObjectStoreResponse setObjectStoreResponse(ObjectStoreResponse storeData, ObjectStoreVO store); Integer countAllObjectStores(); + + Boolean updateAllocatedSize(ObjectStoreVO objectStoreVO, long delta); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDaoImpl.java index 51abde013b69..891ac0996aca 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDaoImpl.java @@ -21,6 +21,10 @@ import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionStatus; + import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.springframework.stereotype.Component; @@ -142,6 +146,19 @@ public ObjectStoreResponse newObjectStoreResponse(ObjectStoreVO store) { ObjectStoreResponse osResponse = new ObjectStoreResponse(); osResponse.setId(store.getUuid()); osResponse.setName(store.getName()); + if (store.getTotalSize() != null && store.getTotalSize() != 0L) { + osResponse.setStorageTotal(store.getTotalSize()); + } + if (store.getUsedSize() == null) { + osResponse.setStorageUsed(0L); + } else { + osResponse.setStorageUsed(store.getUsedSize()); + } + if (store.getAllocatedSize() == null) { + osResponse.setStorageAllocated(0L); + } else { + osResponse.setStorageAllocated(store.getAllocatedSize()); + } osResponse.setProviderName(store.getProviderName()); String url = store.getUrl(); osResponse.setUrl(url); @@ -159,4 +176,19 @@ public Integer countAllObjectStores() { SearchCriteria sc = createSearchCriteria(); return getCount(sc); } + + @Override + public Boolean updateAllocatedSize(ObjectStoreVO objectStoreVO, long delta) { + return Transaction.execute(new TransactionCallback() { + @Override + public Boolean doInTransaction(final TransactionStatus status) { + if (objectStoreVO.getAllocatedSize() != null) { + objectStoreVO.setAllocatedSize(objectStoreVO.getAllocatedSize() + delta); + } else { + objectStoreVO.setAllocatedSize(delta); + } + return update(objectStoreVO.getId(), objectStoreVO); + } + }); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreVO.java index 18cc06a65733..23b650acc79c 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreVO.java @@ -60,8 +60,11 @@ public class ObjectStoreVO implements ObjectStore { @Column(name = "total_size") private Long totalSize; - @Column(name = "used_bytes") - private Long usedBytes; + @Column(name = "used_size") + private Long usedSize; + + @Column(name = "allocated_size") + private Long allocatedSize; @Transient Map details; @@ -130,18 +133,26 @@ public void setTotalSize(Long totalSize) { this.totalSize = totalSize; } - public Long getUsedBytes() { - return usedBytes; + public Long getUsedSize() { + return usedSize; } - public void setUsedBytes(Long usedBytes) { - this.usedBytes = usedBytes; + public void setUsedSize(Long usedSize) { + this.usedSize = usedSize; } public void setDetails(Map details) { this.details = details; } + public Long getAllocatedSize() { + return allocatedSize; + } + + public void setAllocatedSize(Long allocatedSize) { + this.allocatedSize = allocatedSize; + } + @Override public String toString() { return String.format("ObjectStore %s", diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 4d75c35c9337..7e9d6c3b54f0 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -271,6 +271,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index 4fed45c91fba..4677c8bbd3f7 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -21,6 +21,7 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_schedule', 'max_backups', 'INT(8) UNSIGNED NOT NULL DEFAULT 0 COMMENT ''Maximum number of backups to be retained'''); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'backup_schedule_id', 'BIGINT(20) UNSIGNED'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_schedule', 'quiescevm', 'tinyint(1) default NULL COMMENT "Quiesce VM before taking backup"'); -- Update default value for the config 'vm.network.nic.max.secondary.ipaddresses' (and value to default value if value is null) UPDATE `cloud`.`configuration` SET default_value = '10' WHERE name = 'vm.network.nic.max.secondary.ipaddresses'; @@ -663,3 +664,84 @@ ALTER TABLE `cloud`.`networks` MODIFY COLUMN `cidr` varchar(255) DEFAULT NULL CO ALTER TABLE `cloud`.`networks` MODIFY COLUMN `gateway` varchar(255) DEFAULT NULL COMMENT 'gateway(s) for this network configuration'; ALTER TABLE `cloud`.`networks` MODIFY COLUMN `ip6_cidr` varchar(1024) DEFAULT NULL COMMENT 'IPv6 cidr(s) for this network'; ALTER TABLE `cloud`.`networks` MODIFY COLUMN `ip6_gateway` varchar(1024) DEFAULT NULL COMMENT 'IPv6 gateway(s) for this network'; + +-- Add columns name, description and backup_interval_type to backup table +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'name', 'VARCHAR(255) NULL COMMENT "name of the backup"'); +UPDATE `cloud`.`backups` backup INNER JOIN `cloud`.`vm_instance` vm ON backup.vm_id = vm.id SET backup.name = vm.name; +ALTER TABLE `cloud`.`backups` MODIFY COLUMN `name` VARCHAR(255) NOT NULL; +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'description', 'VARCHAR(1024) COMMENT "description for the backup"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'backup_interval_type', 'int(5) COMMENT "type of backup, e.g. manual, recurring - hourly, daily, weekly or monthly"'); + +-- Create backup details table +CREATE TABLE IF NOT EXISTS `cloud`.`backup_details` ( + `id` bigint unsigned NOT NULL auto_increment, + `backup_id` bigint unsigned NOT NULL COMMENT 'backup id', + `name` varchar(255) NOT NULL, + `value` TEXT NOT NULL, + `display` tinyint(1) NOT NULL DEFAULT 1 COMMENT 'Should detail be displayed to the end user', + PRIMARY KEY (`id`), + CONSTRAINT `fk_backup_details__backup_id` FOREIGN KEY `fk_backup_details__backup_id`(`backup_id`) REFERENCES `backups`(`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- Add diskOfferingId, deviceId, minIops and maxIops to backed_volumes in backups table +UPDATE `cloud`.`backups` b +INNER JOIN `cloud`.`vm_instance` vm ON b.vm_id = vm.id +SET b.backed_volumes = ( + SELECT CONCAT("[", + GROUP_CONCAT( + CONCAT( + "{\"uuid\":\"", v.uuid, "\",", + "\"type\":\"", v.volume_type, "\",", + "\"size\":", v.`size`, ",", + "\"path\":\"", IFNULL(v.path, 'null'), "\",", + "\"deviceId\":", IFNULL(v.device_id, 'null'), ",", + "\"diskOfferingId\":\"", doff.uuid, "\",", + "\"minIops\":", IFNULL(v.min_iops, 'null'), ",", + "\"maxIops\":", IFNULL(v.max_iops, 'null'), + "}" + ) + SEPARATOR "," + ), + "]") + FROM `cloud`.`volumes` v + LEFT JOIN `cloud`.`disk_offering` doff ON v.disk_offering_id = doff.id + WHERE v.instance_id = vm.id +); + +-- Add diskOfferingId, deviceId, minIops and maxIops to backup_volumes in vm_instance table +UPDATE `cloud`.`vm_instance` vm +SET vm.backup_volumes = ( + SELECT CONCAT("[", + GROUP_CONCAT( + CONCAT( + "{\"uuid\":\"", v.uuid, "\",", + "\"type\":\"", v.volume_type, "\",", + "\"size\":", v.`size`, ",", + "\"path\":\"", IFNULL(v.path, 'null'), "\",", + "\"deviceId\":", IFNULL(v.device_id, 'null'), ",", + "\"diskOfferingId\":\"", doff.uuid, "\",", + "\"minIops\":", IFNULL(v.min_iops, 'null'), ",", + "\"maxIops\":", IFNULL(v.max_iops, 'null'), + "}" + ) + SEPARATOR "," + ), + "]") + FROM `cloud`.`volumes` v + LEFT JOIN `cloud`.`disk_offering` doff ON v.disk_offering_id = doff.id + WHERE v.instance_id = vm.id +) +WHERE vm.backup_offering_id IS NOT NULL; + +-- Add column allocated_size to object_store table. Rename column 'used_bytes' to 'used_size' +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.object_store', 'allocated_size', 'bigint unsigned COMMENT "allocated size in bytes"'); +ALTER TABLE `cloud`.`object_store` CHANGE COLUMN `used_bytes` `used_size` BIGINT UNSIGNED COMMENT 'used size in bytes'; +ALTER TABLE `cloud`.`object_store` MODIFY COLUMN `total_size` bigint unsigned COMMENT 'total size in bytes'; +UPDATE `cloud`.`object_store` +JOIN ( + SELECT object_store_id, SUM(quota) AS total_quota + FROM `cloud`.`bucket` + WHERE removed IS NULL + GROUP BY object_store_id +) buckets_quota_sum_view ON `object_store`.id = buckets_quota_sum_view.object_store_id +SET `object_store`.allocated_size = buckets_quota_sum_view.total_quota; diff --git a/engine/schema/src/test/java/org/apache/cloudstack/backup/dao/BackupDaoImplTest.java b/engine/schema/src/test/java/org/apache/cloudstack/backup/dao/BackupDaoImplTest.java new file mode 100644 index 000000000000..b12cfb4c14c1 --- /dev/null +++ b/engine/schema/src/test/java/org/apache/cloudstack/backup/dao/BackupDaoImplTest.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup.dao; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.cloudstack.backup.BackupVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class BackupDaoImplTest { + @Spy + @InjectMocks + private BackupDaoImpl backupDao; + + @Mock + BackupDetailsDao backupDetailsDao; + + @Test + public void testLoadDetails() { + Long backupId = 1L; + BackupVO backup = new BackupVO(); + ReflectionTestUtils.setField(backup, "id", backupId); + Map details = new HashMap<>(); + details.put("key1", "value1"); + details.put("key2", "value2"); + + Mockito.when(backupDetailsDao.listDetailsKeyPairs(backupId)).thenReturn(details); + + backupDao.loadDetails(backup); + + Assert.assertEquals(details, backup.getDetails()); + Mockito.verify(backupDetailsDao).listDetailsKeyPairs(backupId); + } + + @Test + public void testSaveDetails() { + Long backupId = 1L; + BackupVO backup = new BackupVO(); + ReflectionTestUtils.setField(backup, "id", backupId); + Map details = new HashMap<>(); + details.put("key1", "value1"); + details.put("key2", "value2"); + backup.setDetails(details); + + backupDao.saveDetails(backup); + + Mockito.verify(backupDetailsDao).saveDetails(Mockito.anyList()); + } +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreHelper.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreHelper.java index c58d801e40ef..a2275576bbef 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreHelper.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreHelper.java @@ -18,6 +18,7 @@ */ package org.apache.cloudstack.storage.object.datastore; +import com.cloud.configuration.Resource; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailVO; @@ -44,6 +45,7 @@ public ObjectStoreVO createObjectStore(Map params, Map restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid, Pair vmNameAndState) { - logger.debug("Restoring volume {} from backup {} on the Dummy Backup Provider", volumeUuid, backup); - throw new CloudRuntimeException("Dummy plugin does not support this feature"); + public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeInfo backupVolumeInfo, String hostIp, String dataStoreUuid, Pair vmNameAndState) { + final VolumeVO volume = volumeDao.findByUuid(backupVolumeInfo.getUuid()); + final StoragePoolHostVO dataStore = storagePoolHostDao.findByUuid(dataStoreUuid); + final DiskOffering diskOffering = diskOfferingDao.findByUuid(backupVolumeInfo.getDiskOfferingId()); + + logger.debug("Restoring volume {} from backup {} on the Dummy Backup Provider", backupVolumeInfo, backup); + + VolumeVO restoredVolume = new VolumeVO(Volume.Type.DATADISK, null, backup.getZoneId(), + backup.getDomainId(), backup.getAccountId(), 0, null, + backup.getSize(), null, null, null); + String volumeUUID = UUID.randomUUID().toString(); + String volumeName = volume != null ? volume.getName() : backupVolumeInfo.getUuid(); + restoredVolume.setName("RestoredVol-" + volumeName); + restoredVolume.setProvisioningType(diskOffering.getProvisioningType()); + restoredVolume.setUpdated(new Date()); + restoredVolume.setUuid(volumeUUID); + restoredVolume.setRemoved(null); + restoredVolume.setDisplayVolume(true); + restoredVolume.setPoolId(dataStore.getPoolId()); + restoredVolume.setPath(restoredVolume.getUuid()); + restoredVolume.setState(Volume.State.Copying); + restoredVolume.setSize(backupVolumeInfo.getSize()); + restoredVolume.setDiskOfferingId(diskOffering.getId()); + + try { + volumeDao.persist(restoredVolume); + } catch (Exception e) { + throw new CloudRuntimeException("Unable to create restored volume due to: " + e); + } + return new Pair<>(true, volumeUUID); } - @Override - public Map getBackupMetrics(Long zoneId, List vms) { - final Map metrics = new HashMap<>(); - final Backup.Metric metric = new Backup.Metric(1000L, 100L); - if (vms == null || vms.isEmpty()) { - return metrics; - } - for (VirtualMachine vm : vms) { - if (vm != null) { - metrics.put(vm, metric); - } - } - return metrics; + public void syncBackupMetrics(Long zoneId) { } @Override @@ -106,7 +132,7 @@ public List listRestorePoints(VirtualMachine vm) { } @Override - public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm) { return null; } @@ -118,11 +144,11 @@ public boolean removeVMFromBackupOffering(VirtualMachine vm) { @Override public boolean willDeleteBackupsOnOfferingRemoval() { - return true; + return false; } @Override - public Pair takeBackup(VirtualMachine vm) { + public Pair takeBackup(VirtualMachine vm, Boolean quiesceVM) { logger.debug("Starting backup for VM {} on Dummy provider", vm); BackupVO backup = new BackupVO(); @@ -130,20 +156,50 @@ public Pair takeBackup(VirtualMachine vm) { backup.setExternalId("dummy-external-id"); backup.setType("FULL"); backup.setDate(new Date()); - backup.setSize(1024000L); - backup.setProtectedSize(Resource.ResourceType.bytesToGiB); + long virtualSize = 0L; + for (final Volume volume: volumeDao.findByInstance(vm.getId())) { + if (Volume.State.Ready.equals(volume.getState())) { + virtualSize += volume.getSize(); + } + } + backup.setSize(virtualSize); + backup.setProtectedSize(virtualSize); backup.setStatus(Backup.Status.BackedUp); backup.setBackupOfferingId(vm.getBackupOfferingId()); backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); - backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); + backup.setName(backupManager.getBackupNameFromVM(vm)); + List volumes = new ArrayList<>(volumeDao.findByInstance(vm.getId())); + backup.setBackedUpVolumes(backupManager.createVolumeInfoFromVolumes(volumes)); + Map details = backupManager.getBackupDetailsFromVM(vm); + backup.setDetails(details); + backup = backupDao.persist(backup); return new Pair<>(true, backup); } @Override public boolean deleteBackup(Backup backup, boolean forced) { + return backupDao.remove(backup.getId()); + } + + @Override + public boolean supportsInstanceFromBackup() { + return true; + } + + @Override + public Pair getBackupStorageStats(Long zoneId) { + return new Pair<>(8L * 1024 * 1024 * 1024, 10L * 1024 * 1024 * 1024); + } + + @Override + public void syncBackupStorageStats(Long zoneId) { + } + + @Override + public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { return true; } } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index f73d82d87c42..e5f98ad291be 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -24,12 +24,15 @@ import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; +import com.cloud.offering.DiskOffering; +import com.cloud.resource.ResourceManager; import com.cloud.storage.ScopeType; import com.cloud.storage.Storage; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiServiceImpl; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.Pair; @@ -50,7 +53,6 @@ import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -63,7 +65,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.HashMap; import java.util.Objects; import java.util.Optional; import java.util.UUID; @@ -78,6 +79,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject private BackupRepositoryDao backupRepositoryDao; + @Inject + private BackupRepositoryService backupRepositoryService; + @Inject private HostDao hostDao; @@ -102,6 +106,15 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject private VMSnapshotDetailsDao vmSnapshotDetailsDao; + @Inject + BackupManager backupManager; + + @Inject + ResourceManager resourceManager; + + @Inject + private DiskOfferingDao diskOfferingDao; + protected Host getLastVMHypervisorHost(VirtualMachine vm) { Long hostId = vm.getLastHostId(); if (hostId == null) { @@ -122,13 +135,7 @@ protected Host getLastVMHypervisorHost(VirtualMachine vm) { } } // Try to find any Host in the zone - for (final HostVO hostInZone : hostDao.listByDataCenterIdAndHypervisorType(host.getDataCenterId(), Hypervisor.HypervisorType.KVM)) { - if (hostInZone.getStatus() == Status.Up) { - LOG.debug("Found Host {} in zone {}", hostInZone, host.getDataCenterId()); - return hostInZone; - } - } - return null; + return resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, host.getDataCenterId()); } protected Host getVMHypervisorHost(VirtualMachine vm) { @@ -150,7 +157,7 @@ protected Host getVMHypervisorHost(VirtualMachine vm) { } @Override - public Pair takeBackup(final VirtualMachine vm) { + public Pair takeBackup(final VirtualMachine vm, Boolean quiesceVM) { final Host host = getVMHypervisorHost(vm); final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(vm.getBackupOfferingId()); @@ -167,6 +174,7 @@ public Pair takeBackup(final VirtualMachine vm) { command.setBackupRepoType(backupRepository.getType()); command.setBackupRepoAddress(backupRepository.getAddress()); command.setMountOptions(backupRepository.getMountOptions()); + command.setQuiesce(quiesceVM); if (VirtualMachine.State.Stopped.equals(vm.getState())) { List vmVolumes = volumeDao.findByInstance(vm.getId()); @@ -179,8 +187,14 @@ public Pair takeBackup(final VirtualMachine vm) { try { answer = (BackupAnswer) agentManager.send(host.getId(), command); } catch (AgentUnavailableException e) { + logger.error("Unable to contact backend control plane to initiate backup for VM {}", vm.getInstanceName()); + backupVO.setStatus(Backup.Status.Failed); + backupDao.remove(backupVO.getId()); throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } catch (OperationTimedoutException e) { + logger.error("Operation to initiate backup timed out for VM {}", vm.getInstanceName()); + backupVO.setStatus(Backup.Status.Failed); + backupDao.remove(backupVO.getId()); throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); } @@ -188,15 +202,23 @@ public Pair takeBackup(final VirtualMachine vm) { backupVO.setDate(new Date()); backupVO.setSize(answer.getSize()); backupVO.setStatus(Backup.Status.BackedUp); - backupVO.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); + List volumes = new ArrayList<>(volumeDao.findByInstance(vm.getId())); + backupVO.setBackedUpVolumes(backupManager.createVolumeInfoFromVolumes(volumes)); if (backupDao.update(backupVO.getId(), backupVO)) { return new Pair<>(true, backupVO); } else { throw new CloudRuntimeException("Failed to update backup"); } } else { - backupVO.setStatus(Backup.Status.Failed); - backupDao.remove(backupVO.getId()); + logger.error("Failed to take backup for VM {}: {}", vm.getInstanceName(), answer != null ? answer.getDetails() : "No answer received"); + if (answer.getNeedsCleanup()) { + logger.error("Backup cleanup failed for VM {}. Leaving the backup in Error state.", vm.getInstanceName()); + backupVO.setStatus(Backup.Status.Error); + backupDao.update(backupVO.getId(), backupVO); + } else { + backupVO.setStatus(Backup.Status.Failed); + backupDao.remove(backupVO.getId()); + } return new Pair<>(false, null); } } @@ -219,19 +241,35 @@ private BackupVO createBackupObject(VirtualMachine vm, String backupPath) { backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); + backup.setName(backupManager.getBackupNameFromVM(vm)); + Map details = backupManager.getBackupDetailsFromVM(vm); + backup.setDetails(details); + return backupDao.persist(backup); } + @Override + public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + return restoreVMBackup(vm, backup); + } + @Override public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { - List backedVolumes = backup.getBackedUpVolumes(); - List volumes = backedVolumes.stream() - .map(volume -> volumeDao.findByUuid(volume.getUuid())) - .sorted((v1, v2) -> Long.compare(v1.getDeviceId(), v2.getDeviceId())) + return restoreVMBackup(vm, backup); + } + + private boolean restoreVMBackup(VirtualMachine vm, Backup backup) { + List backedVolumesUUIDs = backup.getBackedUpVolumes().stream() + .sorted(Comparator.comparingLong(Backup.VolumeInfo::getDeviceId)) + .map(Backup.VolumeInfo::getUuid) + .collect(Collectors.toList()); + + List restoreVolumes = volumeDao.findByInstance(vm.getId()).stream() + .sorted(Comparator.comparingLong(VolumeVO::getDeviceId)) .collect(Collectors.toList()); LOG.debug("Restoring vm {} from backup {} on the NAS Backup Provider", vm, backup); - BackupRepository backupRepository = getBackupRepository(vm, backup); + BackupRepository backupRepository = getBackupRepository(backup); final Host host = getLastVMHypervisorHost(vm); RestoreBackupCommand restoreCommand = new RestoreBackupCommand(); @@ -240,7 +278,8 @@ public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { restoreCommand.setBackupRepoAddress(backupRepository.getAddress()); restoreCommand.setMountOptions(backupRepository.getMountOptions()); restoreCommand.setVmName(vm.getName()); - restoreCommand.setVolumePaths(getVolumePaths(volumes)); + restoreCommand.setBackupVolumesUUIDs(backedVolumesUUIDs); + restoreCommand.setRestoreVolumePaths(getVolumePaths(restoreVolumes)); restoreCommand.setVmExists(vm.getRemoved() == null); restoreCommand.setVmState(vm.getState()); @@ -250,7 +289,7 @@ public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { } catch (AgentUnavailableException e) { throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } catch (OperationTimedoutException e) { - throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); + throw new CloudRuntimeException("Operation to restore backup timed out, please try again"); } return answer.getResult(); } @@ -276,24 +315,22 @@ private List getVolumePaths(List volumes) { } @Override - public Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid, Pair vmNameAndState) { - final VolumeVO volume = volumeDao.findByUuid(volumeUuid); - final VirtualMachine backupSourceVm = vmInstanceDao.findById(backup.getVmId()); + public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeInfo backupVolumeInfo, String hostIp, String dataStoreUuid, Pair vmNameAndState) { + final VolumeVO volume = volumeDao.findByUuid(backupVolumeInfo.getUuid()); + final DiskOffering diskOffering = diskOfferingDao.findByUuid(backupVolumeInfo.getDiskOfferingId()); final StoragePoolHostVO dataStore = storagePoolHostDao.findByUuid(dataStoreUuid); final HostVO hostVO = hostDao.findByIp(hostIp); - Optional matchingVolume = getBackedUpVolumeInfo(backupSourceVm.getBackupVolumeList(), volumeUuid); - Long backedUpVolumeSize = matchingVolume.isPresent() ? matchingVolume.get().getSize() : 0L; - - LOG.debug("Restoring vm volume {} from backup {} on the NAS Backup Provider", volume, backup); - BackupRepository backupRepository = getBackupRepository(backupSourceVm, backup); + LOG.debug("Restoring vm volume {} from backup {} on the NAS Backup Provider", backupVolumeInfo, backup); + BackupRepository backupRepository = getBackupRepository(backup); VolumeVO restoredVolume = new VolumeVO(Volume.Type.DATADISK, null, backup.getZoneId(), backup.getDomainId(), backup.getAccountId(), 0, null, backup.getSize(), null, null, null); String volumeUUID = UUID.randomUUID().toString(); - restoredVolume.setName("RestoredVol-"+volume.getName()); - restoredVolume.setProvisioningType(volume.getProvisioningType()); + String volumeName = volume != null ? volume.getName() : backupVolumeInfo.getUuid(); + restoredVolume.setName("RestoredVol-" + volumeName); + restoredVolume.setProvisioningType(diskOffering.getProvisioningType()); restoredVolume.setUpdated(new Date()); restoredVolume.setUuid(volumeUUID); restoredVolume.setRemoved(null); @@ -302,20 +339,20 @@ public Pair restoreBackedUpVolume(Backup backup, String volumeU restoredVolume.setPath(restoredVolume.getUuid()); restoredVolume.setState(Volume.State.Copying); restoredVolume.setFormat(Storage.ImageFormat.QCOW2); - restoredVolume.setSize(backedUpVolumeSize); - restoredVolume.setDiskOfferingId(volume.getDiskOfferingId()); + restoredVolume.setSize(backupVolumeInfo.getSize()); + restoredVolume.setDiskOfferingId(diskOffering.getId()); RestoreBackupCommand restoreCommand = new RestoreBackupCommand(); restoreCommand.setBackupPath(backup.getExternalId()); restoreCommand.setBackupRepoType(backupRepository.getType()); restoreCommand.setBackupRepoAddress(backupRepository.getAddress()); restoreCommand.setVmName(vmNameAndState.first()); - restoreCommand.setVolumePaths(Collections.singletonList(String.format("%s/%s", dataStore.getLocalPath(), volumeUUID))); - restoreCommand.setDiskType(volume.getVolumeType().name().toLowerCase(Locale.ROOT)); + restoreCommand.setRestoreVolumePaths(Collections.singletonList(String.format("%s/%s", dataStore.getLocalPath(), volumeUUID))); + restoreCommand.setDiskType(backupVolumeInfo.getType().name().toLowerCase(Locale.ROOT)); restoreCommand.setMountOptions(backupRepository.getMountOptions()); restoreCommand.setVmExists(null); restoreCommand.setVmState(vmNameAndState.second()); - restoreCommand.setRestoreVolumeUUID(volumeUuid); + restoreCommand.setRestoreVolumeUUID(backupVolumeInfo.getUuid()); BackupAnswer answer; try { @@ -323,7 +360,7 @@ public Pair restoreBackedUpVolume(Backup backup, String volumeU } catch (AgentUnavailableException e) { throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } catch (OperationTimedoutException e) { - throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); + throw new CloudRuntimeException("Operation to restore backed up volume timed out, please try again"); } if (answer.getResult()) { @@ -337,15 +374,10 @@ public Pair restoreBackedUpVolume(Backup backup, String volumeU return new Pair<>(answer.getResult(), answer.getDetails()); } - private BackupRepository getBackupRepository(VirtualMachine vm, Backup backup) { - BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(vm.getBackupOfferingId()); - final String errorMessage = "No valid backup repository found for the VM, please check the attached backup offering"; - if (backupRepository == null) { - logger.warn(errorMessage + "Re-attempting with the backup offering associated with the backup"); - } - backupRepository = backupRepositoryDao.findByBackupOfferingId(backup.getBackupOfferingId()); + private BackupRepository getBackupRepository(Backup backup) { + BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(backup.getBackupOfferingId()); if (backupRepository == null) { - throw new CloudRuntimeException(errorMessage); + throw new CloudRuntimeException(String.format("No valid backup repository found for the backup %s, please check the attached backup offering", backup.getUuid())); } return backupRepository; } @@ -363,8 +395,13 @@ public boolean deleteBackup(Backup backup, boolean forced) { throw new CloudRuntimeException("No valid backup repository found for the VM, please check the attached backup offering"); } - final VirtualMachine vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); - final Host host = getLastVMHypervisorHost(vm); + final Host host; + final VirtualMachine vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); + if (vm != null) { + host = getLastVMHypervisorHost(vm); + } else { + host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, backup.getZoneId()); + } DeleteBackupCommand command = new DeleteBackupCommand(backup.getExternalId(), backupRepository.getType(), backupRepository.getAddress(), backupRepository.getMountOptions()); @@ -375,7 +412,7 @@ public boolean deleteBackup(Backup backup, boolean forced) { } catch (AgentUnavailableException e) { throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } catch (OperationTimedoutException e) { - throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); + throw new CloudRuntimeException("Operation to delete backup timed out, please try again"); } if (answer != null && answer.getResult()) { @@ -386,30 +423,7 @@ public boolean deleteBackup(Backup backup, boolean forced) { return false; } - @Override - public Map getBackupMetrics(Long zoneId, List vms) { - final Map metrics = new HashMap<>(); - if (CollectionUtils.isEmpty(vms)) { - LOG.warn("Unable to get VM Backup Metrics because the list of VMs is empty."); - return metrics; - } - - for (final VirtualMachine vm : vms) { - Long vmBackupSize = 0L; - Long vmBackupProtectedSize = 0L; - for (final Backup backup: backupDao.listByVmId(null, vm.getId())) { - if (Objects.nonNull(backup.getSize())) { - vmBackupSize += backup.getSize(); - } - if (Objects.nonNull(backup.getProtectedSize())) { - vmBackupProtectedSize += backup.getProtectedSize(); - } - } - Backup.Metric vmBackupMetric = new Backup.Metric(vmBackupSize,vmBackupProtectedSize); - LOG.debug("Metrics for VM {} is [backup size: {}, data size: {}].", vm, vmBackupMetric.getBackupSize(), vmBackupMetric.getDataSize()); - metrics.put(vm, vmBackupMetric); - } - return metrics; + public void syncBackupMetrics(Long zoneId) { } @Override @@ -418,7 +432,7 @@ public List listRestorePoints(VirtualMachine vm) { } @Override - public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm) { return null; } @@ -445,6 +459,45 @@ public boolean willDeleteBackupsOnOfferingRemoval() { return false; } + @Override + public boolean supportsInstanceFromBackup() { + return true; + } + + @Override + public Pair getBackupStorageStats(Long zoneId) { + final List repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); + Long totalSize = 0L; + Long usedSize = 0L; + for (final BackupRepository repository : repositories) { + if (repository.getCapacityBytes() != null) { + totalSize += repository.getCapacityBytes(); + } + if (repository.getUsedBytes() != null) { + usedSize += repository.getUsedBytes(); + } + } + return new Pair<>(usedSize, totalSize); + } + + @Override + public void syncBackupStorageStats(Long zoneId) { + final List repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); + final Host host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, zoneId); + for (final BackupRepository repository : repositories) { + GetBackupStorageStatsCommand command = new GetBackupStorageStatsCommand(repository.getType(), repository.getAddress(), repository.getMountOptions()); + BackupStorageStatsAnswer answer; + try { + answer = (BackupStorageStatsAnswer) agentManager.send(host.getId(), command); + backupRepositoryDao.updateCapacity(repository, answer.getTotalSize(), answer.getUsedSize()); + } catch (AgentUnavailableException e) { + logger.warn("Unable to contact backend control plane to get backup stats for repository: {}", repository.getName()); + } catch (OperationTimedoutException e) { + logger.warn("Operation to get backup stats timed out for the repository: " + repository.getName()); + } + } + } + @Override public List listBackupOfferings(Long zoneId) { final List repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); diff --git a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java new file mode 100644 index 000000000000..d6f29dc1aac1 --- /dev/null +++ b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java @@ -0,0 +1,230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup; + +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupRepositoryDao; +import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.agent.AgentManager; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.resource.ResourceManager; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.Pair; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.VMInstanceDao; + +@RunWith(MockitoJUnitRunner.class) +public class NASBackupProviderTest { + @Spy + @InjectMocks + private NASBackupProvider nasBackupProvider; + + @Mock + private BackupDao backupDao; + + @Mock + private BackupRepositoryDao backupRepositoryDao; + + @Mock + private BackupOfferingDao backupOfferingDao; + + @Mock + private VMInstanceDao vmInstanceDao; + + @Mock + private AgentManager agentManager; + + @Mock + private VolumeDao volumeDao; + + @Mock + private HostDao hostDao; + + @Mock + private BackupManager backupManager; + + @Mock + private ResourceManager resourceManager; + + @Test + public void testDeleteBackup() throws OperationTimedoutException, AgentUnavailableException { + Long hostId = 1L; + BackupVO backup = new BackupVO(); + backup.setBackupOfferingId(1L); + backup.setVmId(1L); + backup.setExternalId("externalId"); + ReflectionTestUtils.setField(backup, "id", 1L); + + BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 1024L); + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + HostVO host = mock(HostVO.class); + Mockito.when(host.getStatus()).thenReturn(Status.Up); + Mockito.when(hostDao.findById(hostId)).thenReturn(host); + Mockito.when(backupRepositoryDao.findByBackupOfferingId(1L)).thenReturn(backupRepository); + Mockito.when(vmInstanceDao.findByIdIncludingRemoved(1L)).thenReturn(vm); + Mockito.when(agentManager.send(anyLong(), Mockito.any(DeleteBackupCommand.class))).thenReturn(new BackupAnswer(new DeleteBackupCommand(null, null, null, null), true, "details")); + Mockito.when(backupDao.remove(1L)).thenReturn(true); + + boolean result = nasBackupProvider.deleteBackup(backup, true); + Assert.assertTrue(result); + } + + @Test + public void testSyncBackupStorageStats() throws AgentUnavailableException, OperationTimedoutException { + BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 1024L); + + HostVO host = mock(HostVO.class); + Mockito.when(resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, 1L)).thenReturn(host); + + Mockito.when(backupRepositoryDao.listByZoneAndProvider(1L, "nas")).thenReturn(Collections.singletonList(backupRepository)); + GetBackupStorageStatsCommand command = new GetBackupStorageStatsCommand("nfs", "address", "sync"); + BackupStorageStatsAnswer answer = new BackupStorageStatsAnswer(command, true, null); + answer.setTotalSize(100L); + answer.setUsedSize(50L); + Mockito.when(agentManager.send(anyLong(), Mockito.any(GetBackupStorageStatsCommand.class))).thenReturn(answer); + + nasBackupProvider.syncBackupStorageStats(1L); + Mockito.verify(backupRepositoryDao, Mockito.times(1)).updateCapacity(backupRepository, 100L, 50L); + } + + @Test + public void testListBackupOfferings() { + BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 1024L); + ReflectionTestUtils.setField(backupRepository, "uuid", "uuid"); + + Mockito.when(backupRepositoryDao.listByZoneAndProvider(1L, "nas")).thenReturn(Collections.singletonList(backupRepository)); + + List result = nasBackupProvider.listBackupOfferings(1L); + Assert.assertEquals(1, result.size()); + Assert.assertEquals("test-repo", result.get(0).getName()); + Assert.assertEquals("uuid", result.get(0).getUuid()); + } + + @Test + public void testGetBackupStorageStats() { + BackupRepositoryVO backupRepository1 = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 1000L); + backupRepository1.setUsedBytes(500L); + + BackupRepositoryVO backupRepository2 = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 2000L); + backupRepository2.setUsedBytes(600L); + + Mockito.when(backupRepositoryDao.listByZoneAndProvider(1L, "nas")) + .thenReturn(List.of(backupRepository1, backupRepository2)); + + Pair result = nasBackupProvider.getBackupStorageStats(1L); + Assert.assertEquals(Long.valueOf(1100L), result.first()); + Assert.assertEquals(Long.valueOf(3000L), result.second()); + } + + @Test + public void takeBackupSuccessfully() throws AgentUnavailableException, OperationTimedoutException { + Long vmId = 1L; + Long hostId = 2L; + Long backupOfferingId = 3L; + Long accountId = 4L; + Long domainId = 5L; + Long zoneId = 6L; + Long backupId = 7L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(vmId); + Mockito.when(vm.getHostId()).thenReturn(hostId); + Mockito.when(vm.getInstanceName()).thenReturn("test-vm"); + Mockito.when(vm.getBackupOfferingId()).thenReturn(backupOfferingId); + Mockito.when(vm.getAccountId()).thenReturn(accountId); + Mockito.when(vm.getDomainId()).thenReturn(domainId); + Mockito.when(vm.getDataCenterId()).thenReturn(zoneId); + Mockito.when(vm.getState()).thenReturn(VMInstanceVO.State.Running); + + BackupRepository backupRepository = mock(BackupRepository.class); + Mockito.when(backupRepository.getType()).thenReturn("nfs"); + Mockito.when(backupRepository.getAddress()).thenReturn("address"); + Mockito.when(backupRepository.getMountOptions()).thenReturn("sync"); + Mockito.when(backupRepositoryDao.findByBackupOfferingId(backupOfferingId)).thenReturn(backupRepository); + + HostVO host = mock(HostVO.class); + Mockito.when(host.getId()).thenReturn(hostId); + Mockito.when(host.getStatus()).thenReturn(Status.Up); + Mockito.when(host.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + Mockito.when(hostDao.findById(hostId)).thenReturn(host); + + VolumeVO volume1 = mock(VolumeVO.class); + Mockito.when(volume1.getState()).thenReturn(Volume.State.Ready); + Mockito.when(volume1.getSize()).thenReturn(100L); + VolumeVO volume2 = mock(VolumeVO.class); + Mockito.when(volume2.getState()).thenReturn(Volume.State.Ready); + Mockito.when(volume2.getSize()).thenReturn(200L); + Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(List.of(volume1, volume2)); + + BackupAnswer answer = mock(BackupAnswer.class); + Mockito.when(answer.getResult()).thenReturn(true); + Mockito.when(answer.getSize()).thenReturn(100L); + Mockito.when(agentManager.send(anyLong(), Mockito.any(TakeBackupCommand.class))).thenReturn(answer); + + Mockito.when(backupDao.persist(Mockito.any(BackupVO.class))).thenAnswer(invocation -> invocation.getArgument(0)); + Mockito.when(backupDao.update(Mockito.anyLong(), Mockito.any(BackupVO.class))).thenReturn(true); + + Pair result = nasBackupProvider.takeBackup(vm, false); + + Assert.assertTrue(result.first()); + Assert.assertNotNull(result.second()); + BackupVO backup = (BackupVO) result.second(); + Assert.assertEquals(Optional.ofNullable(100L), Optional.ofNullable(backup.getSize())); + Assert.assertEquals(Backup.Status.BackedUp, backup.getStatus()); + Assert.assertEquals("FULL", backup.getType()); + Assert.assertEquals(Optional.of(300L), Optional.of(backup.getProtectedSize())); + Assert.assertEquals(Optional.of(backupOfferingId), Optional.of(backup.getBackupOfferingId())); + Assert.assertEquals(Optional.of(accountId), Optional.of(backup.getAccountId())); + Assert.assertEquals(Optional.of(domainId), Optional.of(backup.getDomainId())); + Assert.assertEquals(Optional.of(zoneId), Optional.of(backup.getZoneId())); + + Mockito.verify(backupDao).persist(Mockito.any(BackupVO.class)); + Mockito.verify(backupDao).update(Mockito.anyLong(), Mockito.any(BackupVO.class)); + Mockito.verify(agentManager).send(anyLong(), Mockito.any(TakeBackupCommand.class)); + } +} diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index 504a551bb30b..f39aedb55f2a 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -21,21 +21,27 @@ import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; +import com.cloud.offering.DiskOffering; +import com.cloud.offering.ServiceOffering; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.utils.script.Script; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; import com.cloud.utils.component.AdapterBase; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.ssh.SshHelper; -import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDaoImpl; import org.apache.cloudstack.backup.networker.NetworkerClient; @@ -116,6 +122,18 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid @Inject private VMInstanceDao vmInstanceDao; + @Inject + private VMTemplateDao vmTemplateDao; + + @Inject + ServiceOfferingDao serviceOfferingDao; + + @Inject + private BackupManager backupManager; + + @Inject + private DiskOfferingDao diskOfferingDao; + private static String getUrlDomain(String url) throws URISyntaxException { URI uri; try { @@ -371,10 +389,10 @@ public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { } @Override - public Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid, Pair vmNameAndState) { + public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeInfo backupVolumeInfo, String hostIp, String dataStoreUuid, Pair vmNameAndState) { String networkerServer; - VolumeVO volume = volumeDao.findByUuid(volumeUuid); - VMInstanceVO backupSourceVm = vmInstanceDao.findById(backup.getVmId()); + VolumeVO volume = volumeDao.findByUuid(backupVolumeInfo.getUuid()); + final DiskOffering diskOffering = diskOfferingDao.findByUuid(backupVolumeInfo.getDiskOfferingId()); StoragePoolHostVO dataStore = storagePoolHostDao.findByUuid(dataStoreUuid); HostVO hostVO = hostDao.findByIp(hostIp); @@ -384,9 +402,8 @@ public Pair restoreBackedUpVolume(Backup backup, String volumeU final String SSID = networkerBackup.getShortId(); final String clusterName = networkerBackup.getClientHostname(); final String destinationNetworkerClient = hostVO.getName().split("\\.")[0]; - Long restoredVolumeDiskSize = 0L; - LOG.debug("Restoring volume {} with uuid {} from backup {} on the Networker Backup Provider", volume, volumeUuid, backup); + LOG.debug("Restoring volume {} with uuid {} from backup {} on the Networker Backup Provider", volume, backupVolumeInfo, backup); if ( SSID.isEmpty() ) { LOG.debug("There was an error retrieving the SSID for backup with id " + externalBackupId + " from EMC NEtworker"); @@ -401,18 +418,13 @@ public Pair restoreBackedUpVolume(Backup backup, String volumeU throw new CloudRuntimeException(String.format("Failed to convert API to HOST : %s", e)); } - // Find volume size from backup vols - for ( Backup.VolumeInfo VMVolToRestore : backupSourceVm.getBackupVolumeList()) { - if (VMVolToRestore.getUuid().equals(volumeUuid)) - restoredVolumeDiskSize = (VMVolToRestore.getSize()); - } - VolumeVO restoredVolume = new VolumeVO(Volume.Type.DATADISK, null, backup.getZoneId(), backup.getDomainId(), backup.getAccountId(), 0, null, backup.getSize(), null, null, null); - restoredVolume.setName("RV-"+volume.getName()); - restoredVolume.setProvisioningType(volume.getProvisioningType()); + String volumeName = volume != null ? volume.getName() : backupVolumeInfo.getUuid(); + restoredVolume.setName("RV-" + volumeName); + restoredVolume.setProvisioningType(diskOffering.getProvisioningType()); restoredVolume.setUpdated(new Date()); restoredVolume.setUuid(UUID.randomUUID().toString()); restoredVolume.setRemoved(null); @@ -420,8 +432,8 @@ public Pair restoreBackedUpVolume(Backup backup, String volumeU restoredVolume.setPoolId(volume.getPoolId()); restoredVolume.setPath(restoredVolume.getUuid()); restoredVolume.setState(Volume.State.Copying); - restoredVolume.setSize(restoredVolumeDiskSize); - restoredVolume.setDiskOfferingId(volume.getDiskOfferingId()); + restoredVolume.setSize(backupVolumeInfo.getSize()); + restoredVolume.setDiskOfferingId(diskOffering.getId()); try { volumeDao.persist(restoredVolume); @@ -461,7 +473,7 @@ public Pair restoreBackedUpVolume(Backup backup, String volumeU } @Override - public Pair takeBackup(VirtualMachine vm) { + public Pair takeBackup(VirtualMachine vm, Boolean quiesceVM) { String networkerServer; String clusterName; @@ -511,7 +523,10 @@ public Pair takeBackup(VirtualMachine vm) { LOG.info("EMC Networker finished backup job for vm {} with saveset Time: {}", vm, saveTime); BackupVO backup = getClient(vm.getDataCenterId()).registerBackupForVm(vm, backupJobStart, saveTime); if (backup != null) { - backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); + List volumes = new ArrayList<>(volumeDao.findByInstance(vm.getId())); + backup.setBackedUpVolumes(backupManager.createVolumeInfoFromVolumes(volumes)); + Map details = backupManager.getBackupDetailsFromVM(vm); + backup.setDetails(details); backupDao.persist(backup); return new Pair<>(true, backup); } else { @@ -536,35 +551,11 @@ public boolean deleteBackup(Backup backup, boolean forced) { return false; } - @Override - public Map getBackupMetrics(Long zoneId, List vms) { - final Map metrics = new HashMap<>(); - long vmBackupSize=0L; - long vmBackupProtectedSize=0L; - - if (CollectionUtils.isEmpty(vms)) { - LOG.warn("Unable to get VM Backup Metrics because the list of VMs is empty."); - return metrics; - } - - for (final VirtualMachine vm : vms) { - for ( Backup.VolumeInfo thisVMVol : vm.getBackupVolumeList()) { - vmBackupProtectedSize += (thisVMVol.getSize() / 1024L / 1024L); - } - final ArrayList vmBackups = getClient(zoneId).getBackupsForVm(vm); - for ( String vmBackup : vmBackups ) { - NetworkerBackup vmNwBackup = getClient(zoneId).getNetworkerBackupInfo(vmBackup); - vmBackupSize += vmNwBackup.getSize().getValue() / 1024L; - } - Backup.Metric vmBackupMetric = new Backup.Metric(vmBackupSize,vmBackupProtectedSize); - LOG.debug(String.format("Metrics for VM [%s] is [backup size: %s, data size: %s].", vm, vmBackupMetric.getBackupSize(), vmBackupMetric.getDataSize())); - metrics.put(vm, vmBackupMetric); - } - return metrics; + public void syncBackupMetrics(Long zoneId) { } @Override - public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm) { // Technically an administrator can manually create a backup for a VM by utilizing the KVM scripts // with the proper parameters. So we will register any backups taken on the Networker side from // outside Cloudstack. If ever Networker will support KVM out of the box this functionality also will @@ -597,6 +588,16 @@ public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoi backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); + backup.setName(backupManager.getBackupNameFromVM(vm)); + + HashMap details = new HashMap<>(); + details.put(ApiConstants.HYPERVISOR, vm.getHypervisorType().toString()); + ServiceOffering serviceOffering = serviceOfferingDao.findById(vm.getServiceOfferingId()); + details.put(ApiConstants.SERVICE_OFFERING_ID, serviceOffering.getUuid()); + VirtualMachineTemplate template = vmTemplateDao.findById(vm.getTemplateId()); + details.put(ApiConstants.TEMPLATE_ID, template.getUuid()); + backup.setDetails(details); + backupDao.persist(backup); return backup; } @@ -611,6 +612,25 @@ public List listRestorePoints(VirtualMachine vm) { return backupIds.stream().map(id -> new Backup.RestorePoint(id, null, null)).collect(Collectors.toList()); } + @Override + public boolean supportsInstanceFromBackup() { + return false; + } + + @Override + public Pair getBackupStorageStats(Long zoneId) { + return new Pair<>(0L, 0L); + } + + @Override + public void syncBackupStorageStats(Long zoneId) { + } + @Override public boolean willDeleteBackupsOnOfferingRemoval() { return false; } + + @Override + public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + return true; + } } diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java index 36bfd4564752..271fec78188d 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.backup.BackupOffering; import org.apache.cloudstack.backup.BackupVO; import org.apache.cloudstack.backup.networker.api.NetworkerBackup; @@ -45,6 +46,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import javax.inject.Inject; import javax.net.ssl.SSLContext; import javax.net.ssl.X509TrustManager; import java.io.IOException; @@ -65,6 +67,9 @@ import static org.apache.cloudstack.backup.NetworkerBackupProvider.BACKUP_IDENTIFIER; public class NetworkerClient { + @Inject + BackupManager backupManager; + private static final Logger LOG = LogManager.getLogger(NetworkerClient.class); private final URI apiURI; private final String apiName; @@ -267,6 +272,8 @@ public BackupVO registerBackupForVm(VirtualMachine vm, Date backupJobStart, Stri backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); + backup.setName(backupManager.getBackupNameFromVM(vm)); + return backup; } catch (final IOException e) { LOG.error("Failed to register backup from EMC Networker due to:", e); diff --git a/plugins/backup/networker/src/test/java/org/apache/backup/networker/NetworkerClientTest.java b/plugins/backup/networker/src/test/java/org/apache/backup/networker/NetworkerClientTest.java index 96f8a7a7977f..f8cff7ba1960 100644 --- a/plugins/backup/networker/src/test/java/org/apache/backup/networker/NetworkerClientTest.java +++ b/plugins/backup/networker/src/test/java/org/apache/backup/networker/NetworkerClientTest.java @@ -30,8 +30,11 @@ import java.time.Instant; import java.util.Date; import java.util.List; + import com.cloud.vm.VMInstanceVO; import com.github.tomakehurst.wiremock.client.VerificationException; + +import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.backup.BackupOffering; import org.apache.cloudstack.backup.BackupVO; import org.apache.cloudstack.backup.networker.NetworkerClient; @@ -40,6 +43,9 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; + import com.github.tomakehurst.wiremock.client.BasicCredentials; import com.github.tomakehurst.wiremock.junit.WireMockRule; @@ -58,6 +64,7 @@ public void setUp() throws Exception { .willReturn(aResponse() .withStatus(200))); client = new NetworkerClient(url, adminUsername, adminPassword, false, 60); + ReflectionTestUtils.setField(client, "backupManager", Mockito.mock(BackupManager.class)); } @Test diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index 93278e808513..c81c5d34ea25 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -24,12 +24,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; import javax.inject.Inject; -import org.apache.cloudstack.backup.Backup.Metric; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.veeam.VeeamClient; import org.apache.cloudstack.backup.veeam.api.Job; @@ -45,6 +42,7 @@ import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap; import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; +import com.cloud.storage.Volume; import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; @@ -102,8 +100,12 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, @Inject private VirtualMachineManager virtualMachineManager; @Inject + private BackupManager backupManager; + @Inject private VolumeDao volumeDao; + private Map backupFilesMetricsMap = new HashMap<>(); + protected VeeamClient getClient(final Long zoneId) { try { return new VeeamClient(VeeamUrl.valueIn(zoneId), VeeamVersion.valueIn(zoneId), VeeamUsername.valueIn(zoneId), VeeamPassword.valueIn(zoneId), @@ -213,11 +215,11 @@ public boolean removeVMFromBackupOffering(final VirtualMachine vm) { @Override public boolean willDeleteBackupsOnOfferingRemoval() { - return true; + return false; } @Override - public Pair takeBackup(final VirtualMachine vm) { + public Pair takeBackup(final VirtualMachine vm, Boolean quiesceVM) { final VeeamClient client = getClient(vm.getDataCenterId()); Boolean result = client.startBackupJob(vm.getBackupExternalId()); return new Pair<>(result, null); @@ -244,7 +246,7 @@ public boolean deleteBackup(Backup backup, boolean forced) { client.syncBackupRepository(); - List allBackups = backupDao.listByVmId(backup.getZoneId(), backup.getVmId()); + List allBackups = backupDao.listByVmIdAndOffering(backup.getZoneId(), backup.getVmId(), backup.getBackupOfferingId()); for (Backup b : allBackups) { if (b.getId() != backup.getId()) { backupDao.remove(b.getId()); @@ -289,62 +291,71 @@ private void prepareForBackupRestoration(VirtualMachine vm) { } @Override - public Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid, Pair vmNameAndState) { + public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeInfo backupVolumeInfo, String hostIp, String dataStoreUuid, Pair vmNameAndState) { final Long zoneId = backup.getZoneId(); final String restorePointId = backup.getExternalId(); - return getClient(zoneId).restoreVMToDifferentLocation(restorePointId, hostIp, dataStoreUuid); + return getClient(zoneId).restoreVMToDifferentLocation(restorePointId, null, hostIp, dataStoreUuid); } @Override - public Map getBackupMetrics(final Long zoneId, final List vms) { - final Map metrics = new HashMap<>(); - if (CollectionUtils.isEmpty(vms)) { - logger.warn("Unable to get VM Backup Metrics because the list of VMs is empty."); - return metrics; - } - - List vmUuids = vms.stream().filter(Objects::nonNull).map(VirtualMachine::getUuid).collect(Collectors.toList()); - logger.debug(String.format("Get Backup Metrics for VMs: [%s].", String.join(", ", vmUuids))); - - final Map backendMetrics = getClient(zoneId).getBackupMetrics(); - for (final VirtualMachine vm : vms) { - if (vm == null || !backendMetrics.containsKey(vm.getUuid())) { - continue; - } - - Metric metric = backendMetrics.get(vm.getUuid()); - logger.debug("Metrics for VM [{}] is [backup size: {}, data size: {}].", vm, - metric.getBackupSize(), metric.getDataSize()); - metrics.put(vm, metric); - } - return metrics; + public void syncBackupMetrics(Long zoneId) { + backupFilesMetricsMap = getClient(zoneId).getBackupMetrics(); } @Override - public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm) { BackupVO backup = new BackupVO(); backup.setVmId(vm.getId()); backup.setExternalId(restorePoint.getId()); backup.setType(restorePoint.getType()); backup.setDate(restorePoint.getCreated()); backup.setStatus(Backup.Status.BackedUp); - if (metric != null) { - backup.setSize(metric.getBackupSize()); - backup.setProtectedSize(metric.getDataSize()); + if (restorePoint.getBackupSize() != null) { + backup.setSize(restorePoint.getBackupSize()); + } + if (restorePoint.getDataSize() != null) { + backup.setProtectedSize(restorePoint.getDataSize()); } backup.setBackupOfferingId(vm.getBackupOfferingId()); backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); - backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); + backup.setName(backupManager.getBackupNameFromVM(vm)); + List volumes = new ArrayList<>(volumeDao.findByInstance(vm.getId())); + backup.setBackedUpVolumes(backupManager.createVolumeInfoFromVolumes(volumes)); + Map details = backupManager.getBackupDetailsFromVM(vm); + backup.setDetails(details); backupDao.persist(backup); return backup; } @Override public List listRestorePoints(VirtualMachine vm) { + final VmwareDatacenter vmwareDC = findVmwareDatacenterForVM(vm); String backupName = getGuestBackupName(vm.getInstanceName(), vm.getUuid()); - return getClient(vm.getDataCenterId()).listRestorePoints(backupName, vm.getInstanceName()); + return getClient(vm.getDataCenterId()).listRestorePoints(backupName, vmwareDC.getVcenterHost(), vm.getInstanceName(), backupFilesMetricsMap); + } + + @Override + public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + final Long zoneId = backup.getZoneId(); + final String restorePointId = backup.getExternalId(); + final String restoreLocation = vm.getInstanceName(); + return getClient(zoneId).restoreVMToDifferentLocation(restorePointId, restoreLocation, hostIp, dataStoreUuid).first(); + } + + @Override + public boolean supportsInstanceFromBackup() { + return true; + } + + @Override + public Pair getBackupStorageStats(Long zoneId) { + return new Pair<>(0L, 0L); + } + + @Override + public void syncBackupStorageStats(Long zoneId) { } @Override diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index 9accc0714de9..e2df854f16d1 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -17,8 +17,6 @@ package org.apache.cloudstack.backup.veeam; -import static org.apache.cloudstack.backup.VeeamBackupProvider.BACKUP_IDENTIFIER; - import java.io.IOException; import java.io.InputStream; import java.net.SocketTimeoutException; @@ -109,7 +107,6 @@ public class VeeamClient { private static final String BACKUP_FILE_REFERENCE = "BackupFileReference"; private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); - private String veeamServerIp; private final Integer veeamServerVersion; private String veeamServerUsername; @@ -659,9 +656,7 @@ public boolean setJobSchedule(final String jobName) { public boolean deleteJobAndBackup(final String jobName) { Pair result = executePowerShellCommands(Arrays.asList( String.format("$job = Get-VBRJob -Name '%s'", jobName), - "if ($job) { Remove-VBRJob -Job $job -Confirm:$false }", - String.format("$backup = Get-VBRBackup -Name '%s'", jobName), - "if ($backup) { Remove-VBRBackup -Backup $backup -FromDisk -Confirm:$false }" + "if ($job) { Remove-VBRJob -Job $job -Confirm:$false }" )); return result != null && result.first() && !result.second().contains(FAILED_TO_DELETE); } @@ -721,40 +716,19 @@ protected Map processHttpResponseForBackupMetrics(final I throw new CloudRuntimeException("Could not get backup metrics via Veeam B&R API"); } for (final BackupFile backupFile : backupFiles.getBackupFiles()) { - String vmUuid = null; - String backupName = null; - List links = backupFile.getLink(); - for (Link link : links) { - if (BACKUP_REFERENCE.equals(link.getType())) { - backupName = link.getName(); - break; - } - } - if (backupName != null && backupName.contains(BACKUP_IDENTIFIER)) { - final String[] names = backupName.split(BACKUP_IDENTIFIER); - if (names.length > 1) { - vmUuid = names[1]; - } - } - if (vmUuid == null) { + String backupFileId = StringUtils.substringAfterLast(backupFile.getUid(), ":"); + if (backupFileId.isEmpty()) { continue; } - if (vmUuid.contains(" - ")) { - vmUuid = vmUuid.split(" - ")[0]; - } - Long usedSize = 0L; - Long dataSize = 0L; - if (metrics.containsKey(vmUuid)) { - usedSize = metrics.get(vmUuid).getBackupSize(); - dataSize = metrics.get(vmUuid).getDataSize(); - } + Long backupSize = null; + Long dataSize = null; if (backupFile.getBackupSize() != null) { - usedSize += Long.valueOf(backupFile.getBackupSize()); + backupSize = Long.valueOf(backupFile.getBackupSize()); } if (backupFile.getDataSize() != null) { - dataSize += Long.valueOf(backupFile.getDataSize()); + dataSize = Long.valueOf(backupFile.getDataSize()); } - metrics.put(vmUuid, new Backup.Metric(usedSize, dataSize)); + metrics.put(backupFileId, new Backup.Metric(backupSize, dataSize)); } } catch (final IOException e) { logger.error("Failed to process response to get backup metrics via Veeam B&R API due to:", e); @@ -768,23 +742,14 @@ public Map getBackupMetricsLegacy() { final List cmds = Arrays.asList( "$backups = Get-VBRBackup", "foreach ($backup in $backups) {" + - " $backup.JobName;" + - " $storageGroups = $backup.GetStorageGroups();" + - " foreach ($group in $storageGroups) {" + - " $usedSize = 0;" + - " $dataSize = 0;" + - " $sizePerStorage = $group.GetStorages().Stats.BackupSize;" + - " $dataPerStorage = $group.GetStorages().Stats.DataSize;" + - " foreach ($size in $sizePerStorage) {" + - " $usedSize += $size;" + - " }" + - " foreach ($size in $dataPerStorage) {" + - " $dataSize += $size;" + - " }" + - " $usedSize;" + - " $dataSize;" + + " $restorePoints = Get-VBRRestorePoint -Backup $backup;" + + " foreach ($restorePoint in $restorePoints) {" + + " $backupFile = $restorePoint.GetStorage();" + + " $restorePoint.Id.Guid;" + + " $backupFile.Stats.BackupSize;" + + " $backupFile.Stats.DataSize;" + + " echo \"" + separator + "\";" + " }" + - " echo \"" + separator + "\"" + "}" ); Pair response = executePowerShellCommands(cmds); @@ -796,24 +761,22 @@ public Map getBackupMetricsLegacy() { protected Map processPowerShellResultForBackupMetrics(final String result) { logger.debug("Processing powershell result: " + result); - final String separator = "====="; - final Map sizes = new HashMap<>(); + Map metrics = new HashMap<>(); for (final String block : result.split(separator + "\r\n")) { final String[] parts = block.split("\r\n"); if (parts.length != 3) { continue; } - final String backupName = parts[0]; - if (backupName != null && backupName.contains(BACKUP_IDENTIFIER)) { - final String[] names = backupName.split(BACKUP_IDENTIFIER); - sizes.put(names[names.length - 1], new Backup.Metric(Long.valueOf(parts[1]), Long.valueOf(parts[2]))); - } + final String restorePointId = parts[0]; + final Long backupSize = Long.valueOf(parts[1]); + final Long dataSize = Long.valueOf(parts[2]); + metrics.put(restorePointId, new Backup.Metric(backupSize, dataSize)); } - return sizes; + return metrics; } - private Backup.RestorePoint getRestorePointFromBlock(String[] parts) { + private Backup.RestorePoint getRestorePointFromBlock(String[] parts, Map metricsMap) { logger.debug(String.format("Processing block of restore points: [%s].", StringUtils.join(parts, ", "))); String id = null; Date created = null; @@ -834,10 +797,17 @@ private Backup.RestorePoint getRestorePointFromBlock(String[] parts) { type = split[1].trim(); } } - return new Backup.RestorePoint(id, created, type); + Backup.Metric metric = metricsMap.get(id); + Long backupSize = null; + Long dataSize = null; + if (metric != null) { + backupSize = metric.getBackupSize(); + dataSize = metric.getDataSize(); + } + return new Backup.RestorePoint(id, created, type, backupSize, dataSize); } - public List listRestorePointsLegacy(String backupName, String vmInternalName) { + public List listRestorePointsLegacy(String backupName, String vmInternalName, Map metricsMap) { final List cmds = Arrays.asList( String.format("$backup = Get-VBRBackup -Name '%s'", backupName), String.format("if ($backup) { $restore = (Get-VBRRestorePoint -Backup:$backup -Name \"%s\" ^| Where-Object {$_.IsConsistent -eq $true})", vmInternalName), @@ -855,26 +825,26 @@ public List listRestorePointsLegacy(String backupName, Stri } logger.debug(String.format("Found restore points from [backupName: %s, vmInternalName: %s] which is: [%s].", backupName, vmInternalName, block)); final String[] parts = block.split("\r\n"); - restorePoints.add(getRestorePointFromBlock(parts)); + restorePoints.add(getRestorePointFromBlock(parts, metricsMap)); } return restorePoints; } - public List listRestorePoints(String backupName, String vmInternalName) { + public List listRestorePoints(String backupName, String vmwareDcName, String vmInternalName, Map metricsMap) { if (isLegacyServer()) { - return listRestorePointsLegacy(backupName, vmInternalName); + return listRestorePointsLegacy(backupName, vmInternalName, metricsMap); } else { - return listVmRestorePointsViaVeeamAPI(vmInternalName); + return listVmRestorePointsViaVeeamAPI(vmwareDcName, vmInternalName, metricsMap); } } - public List listVmRestorePointsViaVeeamAPI(String vmInternalName) { + public List listVmRestorePointsViaVeeamAPI(String vmwareDcName, String vmInternalName, Map metricsMap) { logger.debug(String.format("Trying to list VM restore points via Veeam B&R API for VM %s: ", vmInternalName)); try { final HttpResponse response = get(String.format("/vmRestorePoints?format=Entity")); checkResponseOK(response); - return processHttpResponseForVmRestorePoints(response.getEntity().getContent(), vmInternalName); + return processHttpResponseForVmRestorePoints(response.getEntity().getContent(), vmwareDcName, vmInternalName, metricsMap); } catch (final IOException e) { logger.error("Failed to list VM restore points via Veeam B&R API due to:", e); checkResponseTimeOut(e); @@ -882,21 +852,24 @@ public List listVmRestorePointsViaVeeamAPI(String vmInterna return new ArrayList<>(); } - public List processHttpResponseForVmRestorePoints(InputStream content, String vmInternalName) { + public List processHttpResponseForVmRestorePoints(InputStream content, String vmwareDcName, String vmInternalName, Map metricsMap) { List vmRestorePointList = new ArrayList<>(); try { final ObjectMapper objectMapper = new XmlMapper(); final VmRestorePoints vmRestorePoints = objectMapper.readValue(content, VmRestorePoints.class); + final String hierarchyId = findDCHierarchy(vmwareDcName); + final String hierarchyUuid = StringUtils.substringAfterLast(hierarchyId, ":"); if (vmRestorePoints == null) { throw new CloudRuntimeException("Could not get VM restore points via Veeam B&R API"); } for (final VmRestorePoint vmRestorePoint : vmRestorePoints.getVmRestorePoints()) { logger.debug(String.format("Processing VM restore point Name=%s, VmDisplayName=%s for vm name=%s", vmRestorePoint.getName(), vmRestorePoint.getVmDisplayName(), vmInternalName)); - if (!vmInternalName.equals(vmRestorePoint.getVmDisplayName())) { + if (!vmInternalName.equals(vmRestorePoint.getVmDisplayName()) || !vmRestorePoint.getHierarchyObjRef().contains(hierarchyUuid)) { continue; } boolean isReady = true; + String backupFileId = ""; List links = vmRestorePoint.getLink(); for (Link link : links) { if (Arrays.asList(BACKUP_FILE_REFERENCE, RESTORE_POINT_REFERENCE).contains(link.getType()) && !link.getRel().equals("Up")) { @@ -904,15 +877,27 @@ public List processHttpResponseForVmRestorePoints(InputStre isReady = false; break; } + if (link.getType() != null && link.getType().equals(BACKUP_FILE_REFERENCE)) { + backupFileId = StringUtils.substringAfterLast(link.getHref(), "/"); + } } if (!isReady) { continue; } - String vmRestorePointId = vmRestorePoint.getUid().substring(vmRestorePoint.getUid().lastIndexOf(':') + 1); + String vmRestorePointId = StringUtils.substringAfterLast(vmRestorePoint.getUid(), ":"); Date created = formatDate(vmRestorePoint.getCreationTimeUtc()); String type = vmRestorePoint.getPointType(); logger.debug(String.format("Adding restore point %s, %s, %s", vmRestorePointId, created, type)); - vmRestorePointList.add(new Backup.RestorePoint(vmRestorePointId, created, type)); + Long backupSize = null; + Long dataSize = null; + if (!backupFileId.isEmpty()) { + Backup.Metric metric = metricsMap.get(backupFileId); + if (metric != null) { + backupSize = metric.getBackupSize(); + dataSize = metric.getDataSize(); + } + } + vmRestorePointList.add(new Backup.RestorePoint(vmRestorePointId, created, type, backupSize, dataSize)); } } catch (final IOException | ParseException e) { logger.error("Failed to process response to get VM restore points via Veeam B&R API due to:", e); @@ -925,15 +910,17 @@ private Date formatDate(String date) throws ParseException { return dateFormat.parse(StringUtils.substring(date, 0, 19)); } - public Pair restoreVMToDifferentLocation(String restorePointId, String hostIp, String dataStoreUuid) { - final String restoreLocation = RESTORE_VM_SUFFIX + UUID.randomUUID().toString(); + public Pair restoreVMToDifferentLocation(String restorePointId, String restoreLocation, String hostIp, String dataStoreUuid) { + if (restoreLocation == null) { + restoreLocation = RESTORE_VM_SUFFIX + UUID.randomUUID().toString(); + } final String datastoreId = dataStoreUuid.replace("-",""); final List cmds = Arrays.asList( "$points = Get-VBRRestorePoint", - String.format("foreach($point in $points) { if ($point.Id -eq '%s') { break; } }", restorePointId), + String.format("foreach($point in $points) { if ($point.Id -eq '%s') { $restorePoint = $point; break; } }", restorePointId), String.format("$server = Get-VBRServer -Name \"%s\"", hostIp), String.format("$ds = Find-VBRViDatastore -Server:$server -Name \"%s\"", datastoreId), - String.format("$job = Start-VBRRestoreVM -RestorePoint:$point -Server:$server -Datastore:$ds -VMName \"%s\" -RunAsync", restoreLocation), + String.format("$job = Start-VBRRestoreVM -RestorePoint:$restorePoint -Server:$server -Datastore:$ds -VMName \"%s\" -RunAsync", restoreLocation), "while (-not (Get-VBRRestoreSession -Id $job.Id).IsCompleted) { Start-Sleep -Seconds 10 }" ); Pair result = executePowerShellCommands(cmds); diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java index cbfe2fda5929..a82ff551b8a9 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/VeeamBackupProviderTest.java @@ -93,13 +93,14 @@ public void deleteBackupTestSuccessWhenForcedIsTrueAndHasJustOneBackup() { backup.setExternalId("abc"); backup.setType("Full"); backup.setZoneId(3l); + backup.setBackupOfferingId(4l); Mockito.when(vmInstanceDao.findByIdIncludingRemoved(Mockito.anyLong())).thenReturn(vmInstanceVO); Mockito.doReturn(client).when(backupProvider).getClient(2l); Mockito.doReturn(true).when(client).deleteBackup("abc"); List backups = new ArrayList<>(); backups.add(backup); - Mockito.when(backupDao.listByVmId(3l, 1l)).thenReturn(backups); + Mockito.when(backupDao.listByVmIdAndOffering(3l, 1l, 4l)).thenReturn(backups); Mockito.verify(backupDao, Mockito.never()).remove(Mockito.anyLong()); boolean result = backupProvider.deleteBackup(backup, true); assertEquals(true, result); @@ -115,6 +116,7 @@ public void deleteBackupTestSuccessWhenForcedIsTrueAndHasMoreThanOneBackup() { Mockito.when(backup.getVmId()).thenReturn(1l); Mockito.when(backup.getExternalId()).thenReturn("abc"); Mockito.when(backup.getZoneId()).thenReturn(3l); + Mockito.when(backup.getBackupOfferingId()).thenReturn(4l); BackupVO backup2 = Mockito.mock(BackupVO.class); Mockito.when(backup2.getId()).thenReturn(2l); @@ -122,10 +124,7 @@ public void deleteBackupTestSuccessWhenForcedIsTrueAndHasMoreThanOneBackup() { Mockito.when(vmInstanceDao.findByIdIncludingRemoved(Mockito.anyLong())).thenReturn(vmInstanceVO); Mockito.doReturn(client).when(backupProvider).getClient(2l); Mockito.doReturn(true).when(client).deleteBackup("abc"); - List backups = new ArrayList<>(); - backups.add(backup); - backups.add(backup2); - Mockito.when(backupDao.listByVmId(3l, 1l)).thenReturn(backups); + Mockito.when(backupDao.listByVmIdAndOffering(3l, 1l, 4l)).thenReturn(List.of(backup, backup2)); boolean result = backupProvider.deleteBackup(backup, true); Mockito.verify(backupDao, Mockito.times(1)).remove(2l); assertEquals(true, result); diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java index 63d6896bb85c..3485f402417e 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java @@ -209,32 +209,46 @@ public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsFalse() { private void verifyBackupMetrics(Map metrics) { - Assert.assertEquals(2, metrics.size()); + Assert.assertEquals(7, metrics.size()); - Assert.assertTrue(metrics.containsKey("d1bd8abd-fc73-4b77-9047-7be98a2ecb72")); - Assert.assertEquals(537776128L, (long) metrics.get("d1bd8abd-fc73-4b77-9047-7be98a2ecb72").getBackupSize()); - Assert.assertEquals(2147506644L, (long) metrics.get("d1bd8abd-fc73-4b77-9047-7be98a2ecb72").getDataSize()); + Assert.assertTrue(metrics.containsKey("d93d7c7d-068a-4e8f-ba54-e08cea3cb9d2")); + Assert.assertEquals(537776128L, (long) metrics.get("d93d7c7d-068a-4e8f-ba54-e08cea3cb9d2").getBackupSize()); + Assert.assertEquals(2147506644L, (long) metrics.get("d93d7c7d-068a-4e8f-ba54-e08cea3cb9d2").getDataSize()); - Assert.assertTrue(metrics.containsKey("0d752ca6-d628-4d85-a739-75275e4661e6")); - Assert.assertEquals(1268682752L, (long) metrics.get("0d752ca6-d628-4d85-a739-75275e4661e6").getBackupSize()); - Assert.assertEquals(15624049921L, (long) metrics.get("0d752ca6-d628-4d85-a739-75275e4661e6").getDataSize()); + Assert.assertTrue(metrics.containsKey("d2110f5f-aa22-4e67-8084-5d8597f26d63")); + Assert.assertEquals(579756032L, (long) metrics.get("d2110f5f-aa22-4e67-8084-5d8597f26d63").getBackupSize()); + Assert.assertEquals(7516219400L, (long) metrics.get("d2110f5f-aa22-4e67-8084-5d8597f26d63").getDataSize()); } @Test public void testProcessPowerShellResultForBackupMetrics() { - String result = "i-2-3-VM-CSBKP-d1bd8abd-fc73-4b77-9047-7be98a2ecb72\r\n" + + String result = "d93d7c7d-068a-4e8f-ba54-e08cea3cb9d2\r\n" + "537776128\r\n" + "2147506644\r\n" + "=====\r\n" + - "i-13-22-VM-CSBKP-b3b3cb75-cfbf-4496-9c63-a08a93347276\r\n" + + "4b1181fd-7b1e-4af1-a76b-8284a8953b99\r\n" + + "12398592\r\n" + + "71329948\r\n" + "=====\r\n" + - "backup-job-based-on-sla\r\n" + + "8e9a854e-9bb8-4a34-815c-a6ab17a1e72f\r\n" + + "11870208\r\n" + + "72378524\r\n" + "=====\r\n" + - "i-12-20-VM-CSBKP-9f292f11-00ec-4915-84f0-e3895828640e\r\n" + + "7c54d13d-7b9c-465a-8ec8-7a276bde57dd\r\n" + + "12083200\r\n" + + "69232800\r\n" + "=====\r\n" + - "i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\r\n" + - "1268682752\r\n" + - "15624049921\r\n" + + "094564ff-02a1-46c7-b9e5-e249b8b9acf6\r\n" + + "14217216\r\n" + + "76572832\r\n" + + "=====\r\n" + + "1f6f5c49-92ef-4757-b327-e63ae9f1fdea\r\n" + + "12460032\r\n" + + "72378524\r\n" + + "=====\r\n" + + "d2110f5f-aa22-4e67-8084-5d8597f26d63\r\n" + + "579756032\r\n" + + "7516219400\r\n" + "=====\r\n"; Map metrics = client.processPowerShellResultForBackupMetrics(result); @@ -294,22 +308,6 @@ public void testProcessHttpResponseForBackupMetricsForV11() { " 2023-11-01T00:00:35.163Z\n" + " vib\n" + " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-11-04T000109_2AC1.vbk\n" + - " 581083136\n" + - " 7516219404\n" + - " 5.82\n" + - " 2.22\n" + - " 2023-11-04T00:00:24.973Z\n" + - " vbk\n" + - " \n" + " \n" + " \n" + " \n" + @@ -326,54 +324,6 @@ public void testProcessHttpResponseForBackupMetricsForV11() { " 2023-10-28T23:00:33.233Z\n" + " vib\n" + " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-10-30T000022_0CE3.vib\n" + - " 14409728\n" + - " 76572828\n" + - " 1\n" + - " 6.25\n" + - " 2023-10-30T00:00:22.7Z\n" + - " vib\n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-11-06T000018_055B.vib\n" + - " 17883136\n" + - " 80767136\n" + - " 1\n" + - " 5\n" + - " 2023-11-06T00:00:18.253Z\n" + - " vib\n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-11-02T000029_65BE.vib\n" + - " 12521472\n" + - " 72378525\n" + - " 1\n" + - " 6.67\n" + - " 2023-11-02T00:00:29.05Z\n" + - " vib\n" + - " \n" + " \n" + " \n" + " \n" + @@ -463,14 +413,72 @@ public void testGetBackupMetricsViaVeeamAPI() { Map metrics = client.getBackupMetricsViaVeeamAPI(); Assert.assertEquals(1, metrics.size()); - Assert.assertTrue(metrics.containsKey("506760dc-ed77-40d6-a91d-e0914e7a1ad8")); - Assert.assertEquals(535875584L, (long) metrics.get("506760dc-ed77-40d6-a91d-e0914e7a1ad8").getBackupSize()); - Assert.assertEquals(2147507235L, (long) metrics.get("506760dc-ed77-40d6-a91d-e0914e7a1ad8").getDataSize()); + Assert.assertTrue(metrics.containsKey("6bf10cad-9181-45d9-9cc5-dd669366a381")); + Assert.assertEquals(535875584L, (long) metrics.get("6bf10cad-9181-45d9-9cc5-dd669366a381").getBackupSize()); + Assert.assertEquals(2147507235L, (long) metrics.get("6bf10cad-9181-45d9-9cc5-dd669366a381").getDataSize()); } @Test public void testListVmRestorePointsViaVeeamAPI() { - String xmlResponse = "\n" + + String backupFilesXmlResponse = "\n" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-10-28T000059_745D.vbk\n" + + " 579756032\n" + + " 7516219400\n" + + " 5.83\n" + + " 2.22\n" + + " 2023-10-27T23:00:13.74Z\n" + + " vbk\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " V:\\Backup\\i-2-4-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-4-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-1036D2023-11-03T162535_89D6.vbk\n" + + " 12083200\n" + + " 69232800\n" + + " 1\n" + + " 6.67\n" + + " 2023-11-05T00:00:22.827Z\n" + + " vib\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " V:\\Backup\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275e4661e6\\i-2-5-VM-CSBKP-0d752ca6-d628-4d85-a739-75275eD2023-11-01T000035_BEBF.vib\n" + + " 12398592\n" + + " 71329948\n" + + " 1\n" + + " 6.67\n" + + " 2023-11-01T00:00:35.163Z\n" + + " vib\n" + + " \n" + + "\n"; + + wireMockRule.stubFor(get(urlMatching(".*/backupFiles\\?format=Entity")) + .willReturn(aResponse() + .withHeader("content-type", "application/xml") + .withStatus(200) + .withBody(backupFilesXmlResponse))); + + String vmRestorePointsXmlResponse = "\n" + "i-2-4-VM\n" + " Full\n" + " Full\n" + - " urn:VMware:Vm:adb5423b-b578-4c26-8ab8-cde9c1faec55.vm-1036\n" + + " urn:VMware:Vm:24490b30-81db-4038-821f-59694cd89519.vm-1036\n" + " \n" + "\n"; String vmName = "i-2-4-VM"; @@ -498,13 +506,38 @@ public void testListVmRestorePointsViaVeeamAPI() { .willReturn(aResponse() .withHeader("content-type", "application/xml") .withStatus(200) - .withBody(xmlResponse))); - List vmRestorePointList = client.listVmRestorePointsViaVeeamAPI(vmName); + .withBody(vmRestorePointsXmlResponse))); + + String hierarchyXmlResponse = "\n" + + "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " " + + ""; + String vmwareDcName = "10.0.32.153"; + String hierarchyId = "urn:VMware:Vm:24490b30-81db-4038-821f-59694cd89519"; + + wireMockRule.stubFor(get(urlMatching(".*/hierarchyRoots")) + .willReturn(aResponse() + .withHeader("content-type", "application/xml") + .withStatus(200) + .withBody(hierarchyXmlResponse))); + + Map metricsMap = client.getBackupMetrics(); + List vmRestorePointList = client.listVmRestorePointsViaVeeamAPI(vmwareDcName, vmName, metricsMap); Assert.assertEquals(1, vmRestorePointList.size()); Assert.assertEquals("f6d504cf-eafe-4cd2-8dfc-e9cfe2f1e977", vmRestorePointList.get(0).getId()); Assert.assertEquals("2023-11-03 16:26:12", newDateFormat.format(vmRestorePointList.get(0).getCreated())); Assert.assertEquals("Full", vmRestorePointList.get(0).getType()); + Assert.assertEquals(12083200L, (long) vmRestorePointList.get(0).getBackupSize()); + Assert.assertEquals(69232800L, (long) vmRestorePointList.get(0).getDataSize()); } @Test diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetBackupStatsCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetBackupStatsCommandWrapper.java new file mode 100644 index 000000000000..9eb8cb71e3a7 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetBackupStatsCommandWrapper.java @@ -0,0 +1,71 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.cloudstack.backup.BackupStorageStatsAnswer; +import org.apache.cloudstack.backup.GetBackupStorageStatsCommand; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.Pair; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = GetBackupStorageStatsCommand.class) +public class LibvirtGetBackupStatsCommandWrapper extends CommandWrapper { + @Override + public Answer execute(GetBackupStorageStatsCommand command, LibvirtComputingResource libvirtComputingResource) { + final String backupRepoType = command.getBackupRepoType(); + final String backupRepoAddress = command.getBackupRepoAddress(); + final String mountOptions = command.getMountOptions(); + + List commands = new ArrayList<>(); + commands.add(new String[]{ + libvirtComputingResource.getNasBackupPath(), + "-o", "stats", + "-t", backupRepoType, + "-s", backupRepoAddress, + "-m", mountOptions + }); + + Pair result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); + + logger.debug(String.format("Get backup storage stats result: %s , exit code: %s", result.second(), result.first())); + + if (result.first() != 0) { + logger.debug(String.format("Failed to get backup storage stats: %s", result.second())); + return new BackupStorageStatsAnswer(command, false, result.second()); + } + + BackupStorageStatsAnswer answer = new BackupStorageStatsAnswer(command, false, result.second()); + + String [] stats = result.second().split("\\s+"); + Long total = Long.parseLong(stats[1]) * 1024; + Long used = Long.parseLong(stats[2]) * 1024; + answer.setTotalSize(total); + answer.setUsedSize(used); + + return answer; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 8abc359250c8..0e5091ebcf49 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -58,21 +58,22 @@ public Answer execute(RestoreBackupCommand command, LibvirtComputingResource ser String mountOptions = command.getMountOptions(); Boolean vmExists = command.isVmExists(); String diskType = command.getDiskType(); - List volumePaths = command.getVolumePaths(); + List backedVolumeUUIDs = command.getBackupVolumesUUIDs(); + List restoreVolumePaths = command.getRestoreVolumePaths(); String restoreVolumeUuid = command.getRestoreVolumeUUID(); String newVolumeId = null; try { if (Objects.isNull(vmExists)) { - String volumePath = volumePaths.get(0); + String volumePath = restoreVolumePaths.get(0); int lastIndex = volumePath.lastIndexOf("/"); newVolumeId = volumePath.substring(lastIndex + 1); restoreVolume(backupPath, backupRepoType, backupRepoAddress, volumePath, diskType, restoreVolumeUuid, new Pair<>(vmName, command.getVmState()), mountOptions); } else if (Boolean.TRUE.equals(vmExists)) { - restoreVolumesOfExistingVM(volumePaths, backupPath, backupRepoType, backupRepoAddress, mountOptions); + restoreVolumesOfExistingVM(restoreVolumePaths, backedVolumeUUIDs, backupPath, backupRepoType, backupRepoAddress, mountOptions); } else { - restoreVolumesOfDestroyedVMs(volumePaths, vmName, backupPath, backupRepoType, backupRepoAddress, mountOptions); + restoreVolumesOfDestroyedVMs(restoreVolumePaths, vmName, backupPath, backupRepoType, backupRepoAddress, mountOptions); } } catch (CloudRuntimeException e) { String errorMessage = "Failed to restore backup for VM: " + vmName + "."; @@ -86,24 +87,24 @@ public Answer execute(RestoreBackupCommand command, LibvirtComputingResource ser return new BackupAnswer(command, true, newVolumeId); } - private void restoreVolumesOfExistingVM(List volumePaths, String backupPath, - String backupRepoType, String backupRepoAddress, String mountOptions) { + private void restoreVolumesOfExistingVM(List restoreVolumePaths, List backedVolumesUUIDs, String backupPath, + String backupRepoType, String backupRepoAddress, String mountOptions) { String diskType = "root"; String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions); try { - for (int idx = 0; idx < volumePaths.size(); idx++) { - String volumePath = volumePaths.get(idx); - Pair bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, null); + for (int idx = 0; idx < restoreVolumePaths.size(); idx++) { + String restoreVolumePath = restoreVolumePaths.get(idx); + String backupVolumeUuid = backedVolumesUUIDs.get(idx); + Pair bkpPathAndVolUuid = getBackupPath(mountDirectory, null, backupPath, diskType, backupVolumeUuid); diskType = "datadisk"; - if (!replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first())) { - throw new CloudRuntimeException(String.format("Unable to restore backup for volume [%s].", bkpPathAndVolUuid.second())); + if (!replaceVolumeWithBackup(restoreVolumePath, bkpPathAndVolUuid.first())) { + throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", bkpPathAndVolUuid.second())); } } } finally { unmountBackupDirectory(mountDirectory); deleteTemporaryDirectory(mountDirectory); } - } private void restoreVolumesOfDestroyedVMs(List volumePaths, String vmName, String backupPath, @@ -116,7 +117,7 @@ private void restoreVolumesOfDestroyedVMs(List volumePaths, String vmNam Pair bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, null); diskType = "datadisk"; if (!replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first())) { - throw new CloudRuntimeException(String.format("Unable to restore backup for volume [%s].", bkpPathAndVolUuid.second())); + throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", bkpPathAndVolUuid.second())); } } } finally { @@ -132,7 +133,7 @@ private void restoreVolume(String backupPath, String backupRepoType, String back try { bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, volumeUUID); if (!replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first())) { - throw new CloudRuntimeException(String.format("Unable to restore backup for volume [%s].", bkpPathAndVolUuid.second())); + throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", bkpPathAndVolUuid.second())); } if (VirtualMachine.State.Running.equals(vmNameAndState.second())) { if (!attachVolumeToVm(vmNameAndState.first(), volumePath)) { @@ -190,8 +191,7 @@ private void deleteTemporaryDirectory(String backupDirectory) { private Pair getBackupPath(String mountDirectory, String volumePath, String backupPath, String diskType, String volumeUuid) { String bkpPath = String.format(FILE_PATH_PLACEHOLDER, mountDirectory, backupPath); - int lastIndex = volumePath.lastIndexOf(File.separator); - String volUuid = Objects.isNull(volumeUuid) ? volumePath.substring(lastIndex + 1) : volumeUuid; + String volUuid = Objects.isNull(volumeUuid) ? volumePath.substring(volumePath.lastIndexOf(File.separator) + 1) : volumeUuid; String backupFileName = String.format("%s.%s.qcow2", diskType.toLowerCase(Locale.ROOT), volUuid); bkpPath = String.format(FILE_PATH_PLACEHOLDER, bkpPath, backupFileName); return new Pair<>(bkpPath, volUuid); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java index 3c0cc53bb73b..c7a67080fbfe 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java @@ -36,6 +36,7 @@ @ResourceWrapper(handles = TakeBackupCommand.class) public class LibvirtTakeBackupCommandWrapper extends CommandWrapper { + private static final Integer EXIT_CLEANUP_FAILED = 20; @Override public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvirtComputingResource) { final String vmName = command.getVmName(); @@ -54,6 +55,7 @@ public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvir "-s", backupRepoAddress, "-m", Objects.nonNull(mountOptions) ? mountOptions : "", "-p", backupPath, + "-q", command.getQuiesce() != null && command.getQuiesce() ? "true" : "false", "-d", (Objects.nonNull(diskPaths) && !diskPaths.isEmpty()) ? String.join(",", diskPaths) : "" }); @@ -61,7 +63,12 @@ public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvir if (result.first() != 0) { logger.debug("Failed to take VM backup: " + result.second()); - return new BackupAnswer(command, false, result.second().trim()); + BackupAnswer answer = new BackupAnswer(command, false, result.second().trim()); + if (result.first() == EXIT_CLEANUP_FAILED) { + logger.debug("Backup cleanup failed"); + answer.setNeedsCleanup(true); + } + return answer; } long backupSize = 0L; diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/simulator/SimulatorGuru.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/simulator/SimulatorGuru.java index 246d86d2712d..aae28c428b8d 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/simulator/SimulatorGuru.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/simulator/SimulatorGuru.java @@ -17,6 +17,7 @@ package com.cloud.simulator; import java.util.Date; +import java.util.List; import java.util.Map; import javax.inject.Inject; @@ -35,8 +36,8 @@ import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; -import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.dao.NicDao; public class SimulatorGuru extends HypervisorGuruBase implements HypervisorGuru { @Inject @@ -90,6 +91,29 @@ public VirtualMachine importVirtualMachineFromBackup(long zoneId, long domainId, return vm; } + @Override + public boolean attachRestoredVolumeToVirtualMachine(long zoneId, String location, Backup.VolumeInfo volumeInfo, VirtualMachine vm, long poolId, Backup backup) { + + VMInstanceVO targetVM = instanceDao.findVMByInstanceNameIncludingRemoved(vm.getName()); + List vmVolumes = volumeDao.findByInstance(targetVM.getId()); + VolumeVO restoredVolume = volumeDao.findByUuid(location); + if (restoredVolume != null) { + try { + volumeDao.attachVolume(restoredVolume.getId(), vm.getId(), getNextAvailableDeviceId(vmVolumes)); + restoredVolume.setState(Volume.State.Ready); + volumeDao.update(restoredVolume.getId(), restoredVolume); + return true; + } catch (Exception e) { + restoredVolume.setDisplay(false); + restoredVolume.setDisplayVolume(false); + restoredVolume.setState(Volume.State.Destroy); + volumeDao.update(restoredVolume.getId(), restoredVolume); + throw new RuntimeException("Unable to attach volume " + restoredVolume.getName() + " to VM" + vm.getName() + " due to : " + e.getMessage()); + } + } + return false; + } + @Override public boolean trackVmHostChange() { return false; @@ -100,4 +124,11 @@ public Map getClusterSettings(long vmId) { return null; } + private long getNextAvailableDeviceId(List vmVolumes) { + if (vmVolumes == null || vmVolumes.isEmpty()) { + return 0; + } + long maxDeviceId = vmVolumes.stream() .mapToLong(VolumeVO::getDeviceId) .max() .orElse(-1); + return maxDeviceId + 1; + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index 8f4d7e4da925..88df637b0dd1 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -870,7 +870,9 @@ protected String createVolumeInfoFromVolumes(List vmVolumes) { try { List list = new ArrayList<>(); for (VolumeVO vol : vmVolumes) { - list.add(new Backup.VolumeInfo(vol.getUuid(), vol.getPath(), vol.getVolumeType(), vol.getSize())); + DiskOfferingVO diskOffering = diskOfferingDao.findById(vol.getDiskOfferingId()); + list.add(new Backup.VolumeInfo(vol.getUuid(), vol.getPath(), vol.getVolumeType(), vol.getSize(), + vol.getDeviceId(), diskOffering.getUuid(), vol.getMinIops(), vol.getMaxIops())); } return GSON.toJson(list.toArray(), Backup.VolumeInfo[].class); } catch (Exception e) { diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VMwareGuruTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VMwareGuruTest.java index 4da513db3e4e..6e96330ac584 100644 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VMwareGuruTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VMwareGuruTest.java @@ -69,12 +69,14 @@ import com.cloud.hypervisor.vmware.util.VmwareClient; import com.cloud.hypervisor.vmware.util.VmwareContext; import com.cloud.hypervisor.vmware.util.VmwareHelper; +import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.Storage; import com.cloud.storage.Storage.ProvisioningType; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.utils.Pair; import com.cloud.utils.UuidUtils; @@ -110,6 +112,9 @@ public class VMwareGuruTest { @Mock ClusterDetailsDao _clusterDetailsDao; + @Mock + DiskOfferingDao diskOfferingDao; + AutoCloseable closeable; @Mock @@ -187,19 +192,26 @@ public void createVolumeInfoFromVolumesTestNullVolume() { @Test public void createVolumeInfoFromVolumesTestCorrectlyConvertOfVolumes() { + Long diskOfferingId = 5L; + DiskOfferingVO diskOffering = Mockito.mock(DiskOfferingVO.class); + Mockito.when(diskOffering.getUuid()).thenReturn("disk-offering-uuid"); + Mockito.when(diskOfferingDao.findById(diskOfferingId)).thenReturn(diskOffering); + List volumesToTest = new ArrayList<>(); - VolumeVO root = new VolumeVO("test", 1l, 1l, 1l, 1l, 1l, "test", "/root/dir", ProvisioningType.THIN, 555l, Volume.Type.ROOT); + VolumeVO root = new VolumeVO("test", 1l, 1l, 1l, 1l, 6l, "test", "/root/dir", ProvisioningType.THIN, 555l, Volume.Type.ROOT); + root.setDiskOfferingId(diskOfferingId); String rootUuid = root.getUuid(); VolumeVO data = new VolumeVO("test", 1l, 1l, 1l, 1l, 1l, "test", "/root/dir/data", ProvisioningType.THIN, 1111000l, Volume.Type.DATADISK); + data.setDiskOfferingId(diskOfferingId); String dataUuid = data.getUuid(); volumesToTest.add(root); volumesToTest.add(data); String result = vMwareGuru.createVolumeInfoFromVolumes(volumesToTest); - String expected = String.format("[{\"uuid\":\"%s\",\"type\":\"ROOT\",\"size\":555,\"path\":\"/root/dir\"},{\"uuid\":\"%s\",\"type\":\"DATADISK\",\"size\":1111000,\"path\":\"/root/dir/data\"}]", rootUuid, dataUuid); + String expected = String.format("[{\"uuid\":\"%s\",\"type\":\"ROOT\",\"size\":555,\"path\":\"/root/dir\",\"diskOfferingId\":\"disk-offering-uuid\"},{\"uuid\":\"%s\",\"type\":\"DATADISK\",\"size\":1111000,\"path\":\"/root/dir/data\",\"diskOfferingId\":\"disk-offering-uuid\"}]", rootUuid, dataUuid); assertEquals(expected, result); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 5546b38a0605..9bad38c2af3a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -421,13 +421,13 @@ protected UserVm createKubernetesNode(String joinIp, Long domainId, Long account List securityGroupIds = new ArrayList<>(); securityGroupIds.add(kubernetesCluster.getSecurityGroupId()); nodeVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, workerNodeTemplate, networkIds, securityGroupIds, owner, - hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs, + hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs, null, addrs, null, null, Objects.nonNull(affinityGroupId) ? Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, null, UserVmManager.CKS_NODE, null, null); } else { nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, workerNodeTemplate, networkIds, owner, - hostName, hostName, null, null, null, + hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs, null, addrs, null, null, Objects.nonNull(affinityGroupId) ? Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 21db2ebb8f74..9ffee220a10f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -276,13 +276,13 @@ private Pair createKubernetesControlNode(final Network network, S List securityGroupIds = new ArrayList<>(); securityGroupIds.add(kubernetesCluster.getSecurityGroupId()); controlVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, securityGroupIds, owner, - hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, userDataId, userDataDetails, keypairs, + hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, userDataId, userDataDetails, keypairs, requestedIps, addrs, null, null, Objects.nonNull(affinityGroupId) ? Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, null, UserVmManager.CKS_NODE, null, null); } else { controlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, owner, - hostName, hostName, null, null, null, + hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, userDataId, userDataDetails, keypairs, requestedIps, addrs, null, null, Objects.nonNull(affinityGroupId) ? Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null); @@ -444,13 +444,13 @@ private UserVm createKubernetesAdditionalControlNode(final String joinIp, final List securityGroupIds = new ArrayList<>(); securityGroupIds.add(kubernetesCluster.getSecurityGroupId()); additionalControlVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, securityGroupIds, owner, - hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs, + hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs, null, addrs, null, null, Objects.nonNull(affinityGroupId) ? Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, null, UserVmManager.CKS_NODE, null, null); } else { additionalControlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, owner, - hostName, hostName, null, null, null, + hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs, null, addrs, null, null, Objects.nonNull(affinityGroupId) ? Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null); @@ -488,13 +488,13 @@ private UserVm createEtcdNode(List requestedIps, List securityGroupIds = new ArrayList<>(); securityGroupIds.add(kubernetesCluster.getSecurityGroupId()); etcdNode = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, etcdTemplate, networkIds, securityGroupIds, owner, - hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs, + hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs, Map.of(kubernetesCluster.getNetworkId(), requestedIps.get(etcdNodeIndex)), addrs, null, null, Objects.nonNull(affinityGroupId) ? Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, null, null, null, null); } else { etcdNode = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, etcdTemplate, networkIds, owner, - hostName, hostName, null, null, null, + hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs, Map.of(kubernetesCluster.getNetworkId(), requestedIps.get(etcdNodeIndex)), addrs, null, null, Objects.nonNull(affinityGroupId) ? Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null); diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java index 8740d188ce07..f0b0ecf5c61a 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java @@ -54,6 +54,7 @@ public CephObjectStoreLifeCycleImpl() { public DataStore initialize(Map dsInfos) { String url = (String)dsInfos.get("url"); String name = (String)dsInfos.get("name"); + Long size = (Long)dsInfos.get("size"); String providerName = (String)dsInfos.get("providerName"); Map details = (Map)dsInfos.get("details"); if (details == null) { @@ -67,6 +68,7 @@ public DataStore initialize(Map dsInfos) { Map objectStoreParameters = new HashMap(); objectStoreParameters.put("name", name); objectStoreParameters.put("url", url); + objectStoreParameters.put("size", size); objectStoreParameters.put("providerName", providerName); objectStoreParameters.put("accesskey", accessKey); diff --git a/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/MinIOObjectStoreLifeCycleImpl.java b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/MinIOObjectStoreLifeCycleImpl.java index 9d620b32b544..1fb0d5159494 100644 --- a/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/MinIOObjectStoreLifeCycleImpl.java +++ b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/MinIOObjectStoreLifeCycleImpl.java @@ -53,6 +53,7 @@ public DataStore initialize(Map dsInfos) { String url = (String)dsInfos.get("url"); String name = (String)dsInfos.get("name"); + Long size = (Long)dsInfos.get("size"); String providerName = (String)dsInfos.get("providerName"); Map details = (Map)dsInfos.get("details"); if(details == null){ @@ -65,6 +66,7 @@ public DataStore initialize(Map dsInfos) { Map objectStoreParameters = new HashMap(); objectStoreParameters.put("name", name); objectStoreParameters.put("url", url); + objectStoreParameters.put("size", size); objectStoreParameters.put("providerName", providerName); objectStoreParameters.put("accesskey", accessKey); diff --git a/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorObjectStoreLifeCycleImpl.java b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorObjectStoreLifeCycleImpl.java index 6ceed041e8df..3c5bc76ba952 100644 --- a/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorObjectStoreLifeCycleImpl.java +++ b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorObjectStoreLifeCycleImpl.java @@ -65,12 +65,14 @@ public DataStore initialize(Map dsInfos) { String url = (String)dsInfos.get("url"); String name = (String)dsInfos.get("name"); String providerName = (String)dsInfos.get("providerName"); + Long size = (Long)dsInfos.get("size"); Map details = (Map)dsInfos.get("details"); Map objectStoreParameters = new HashMap(); objectStoreParameters.put("name", name); objectStoreParameters.put("url", url); objectStoreParameters.put("providerName", providerName); + objectStoreParameters.put("size", size); ObjectStoreVO ids = objectStoreHelper.createObjectStore(objectStoreParameters, details); return objectStoreMgr.getObjectStore(ids.getId()); diff --git a/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java b/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java index a625761ec283..c1d4153fbd1c 100644 --- a/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java +++ b/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java @@ -195,7 +195,7 @@ private UserVm deploySharedFSVM(Long zoneId, Account owner, List networkId CallContext vmContext = CallContext.register(CallContext.current(), ApiCommandResourceType.VirtualMachine); try { vm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, hostName, hostName, - diskOfferingId, size, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, + diskOfferingId, size, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs, null, addrs, null, null, null, customParameterMap, null, null, null, null, true, UserVmManager.SHAREDFSVM, null, null, null); diff --git a/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java b/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java index 21753257f750..5f30dab383f5 100644 --- a/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java +++ b/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java @@ -254,7 +254,7 @@ public void testDeploySharedFS() throws ResourceUnavailableException, Insufficie when(vm.getId()).thenReturn(s_vmId); when(userVmService.createAdvancedVirtualMachine( any(DataCenter.class), any(ServiceOffering.class), any(VirtualMachineTemplate.class), anyList(), any(Account.class), anyString(), - anyString(), anyLong(), anyLong(), isNull(), any(Hypervisor.HypervisorType.class), any(BaseCmd.HTTPMethod.class), anyString(), + anyString(), anyLong(), anyLong(), any(), isNull(), any(Hypervisor.HypervisorType.class), any(BaseCmd.HTTPMethod.class), anyString(), isNull(), isNull(), anyList(), isNull(), any(Network.IpAddresses.class), isNull(), isNull(), isNull(), anyMap(), isNull(), isNull(), isNull(), isNull(), anyBoolean(), anyString(), isNull(), isNull(), isNull())).thenReturn(vm); diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index 9dedaef154a3..588c3791769e 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -16,7 +16,7 @@ ## specific language governing permissions and limitations ## under the License. -set -e +set -eo pipefail # CloudStack B&R NAS Backup and Recovery Tool for KVM @@ -31,8 +31,11 @@ NAS_ADDRESS="" MOUNT_OPTS="" BACKUP_DIR="" DISK_PATHS="" +QUIESCE="" logFile="/var/log/cloudstack/agent/agent.log" +EXIT_CLEANUP_FAILED=20 + log() { [[ "$verb" -eq 1 ]] && builtin echo "$@" if [[ "$1" == "-ne" || "$1" == "-e" || "$1" == "-n" ]]; then @@ -88,7 +91,7 @@ sanity_checks() { backup_running_vm() { mount_operation - mkdir -p $dest + mkdir -p "$dest" || { echo "Failed to create backup directory $dest"; exit 1; } name="root" echo "" > $dest/backup.xml @@ -99,8 +102,31 @@ backup_running_vm() { done echo "" >> $dest/backup.xml + local thaw=0 + if [[ ${QUIESCE} == "true" ]]; then + if virsh -c qemu:///system qemu-agent-command "$VM" '{"execute":"guest-fsfreeze-freeze"}' > /dev/null 2>/dev/null; then + thaw=1 + fi + fi + # Start push backup - virsh -c qemu:///system backup-begin --domain $VM --backupxml $dest/backup.xml > /dev/null 2>/dev/null + local backup_begin=0 + if virsh -c qemu:///system backup-begin --domain $VM --backupxml $dest/backup.xml 2>&1 > /dev/null; then + backup_begin=1; + fi + + if [[ $thaw -eq 1 ]]; then + if ! response=$(virsh -c qemu:///system qemu-agent-command "$VM" '{"execute":"guest-fsfreeze-thaw"}' 2>&1 > /dev/null); then + echo "Failed to thaw the filesystem for vm $VM: $response" + cleanup + exit 1 + fi + fi + + if [[ $backup_begin -ne 1 ]]; then + cleanup + exit 1 + fi # Backup domain information virsh -c qemu:///system dumpxml $VM > $dest/domain-config.xml 2>/dev/null @@ -108,9 +134,18 @@ backup_running_vm() { virsh -c qemu:///system domiflist $VM > $dest/domiflist.xml 2>/dev/null virsh -c qemu:///system domblklist $VM > $dest/domblklist.xml 2>/dev/null - until virsh -c qemu:///system domjobinfo $VM --completed --keep-completed 2>/dev/null | grep "Completed" > /dev/null; do + while true; do + status=$(virsh -c qemu:///system domjobinfo $VM --completed --keep-completed | awk '/Job type:/ {print $3}') + case "$status" in + Completed) + break ;; + Failed) + echo "Virsh backup job failed" + cleanup ;; + esac sleep 5 done + rm -f $dest/backup.xml sync @@ -124,14 +159,18 @@ backup_running_vm() { backup_stopped_vm() { mount_operation - mkdir -p $dest + mkdir -p "$dest" || { echo "Failed to create backup directory $dest"; exit 1; } IFS="," name="root" for disk in $DISK_PATHS; do volUuid="${disk##*/}" - qemu-img convert -O qcow2 $disk $dest/$name.$volUuid.qcow2 | tee -a "$logFile" + output="$dest/$name.$volUuid.qcow2" + if ! qemu-img convert -O qcow2 "$disk" "$output" > "$logFile" 2> >(cat >&2); then + echo "qemu-img convert failed for $disk $output" + cleanup + fi name="datadisk" done sync @@ -148,13 +187,22 @@ delete_backup() { rmdir $mount_point } +get_backup_stats() { + mount_operation + + echo $mount_point + df -P $mount_point 2>/dev/null | awk 'NR==2 {print $2, $3}' + umount $mount_point + rmdir $mount_point +} + mount_operation() { mount_point=$(mktemp -d -t csbackup.XXXXX) dest="$mount_point/${BACKUP_DIR}" if [ ${NAS_TYPE} == "cifs" ]; then MOUNT_OPTS="${MOUNT_OPTS},nobrl" fi - mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS}) | tee -a "$logFile" + mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS}) 2>&1 | tee -a "$logFile" if [ $? -eq 0 ]; then log -ne "Successfully mounted ${NAS_TYPE} store" else @@ -163,9 +211,22 @@ mount_operation() { fi } +cleanup() { + local status=0 + + rm -rf "$dest" || { echo "Failed to delete $dest"; status=1; } + umount "$mount_point" || { echo "Failed to unmount $mount_point"; status=1; } + rmdir "$mount_point" || { echo "Failed to remove mount point $mount_point"; status=1; } + + if [[ $status -ne 0 ]]; then + echo "Backup cleanup failed" + exit $EXIT_CLEANUP_FAILED + fi +} + function usage { echo "" - echo "Usage: $0 -o -v|--vm -t -s -m -p -d " + echo "Usage: $0 -o -v|--vm -t -s -m -p -d -q|--quiesce " echo "" exit 1 } @@ -202,6 +263,11 @@ while [[ $# -gt 0 ]]; do shift shift ;; + -q|--quiesce) + QUIESCE="$2" + shift + shift + ;; -d|--diskpaths) DISK_PATHS="$2" shift @@ -222,12 +288,14 @@ done sanity_checks if [ "$OP" = "backup" ]; then - STATE=$(virsh -c qemu:///system list | grep $VM | awk '{print $3}') - if [ "$STATE" = "running" ]; then + STATE=$(virsh -c qemu:///system list | awk -v vm="$VM" '$2 == vm {print $3}') + if [ -n "$STATE" ] && [ "$STATE" = "running" ]; then backup_running_vm else backup_stopped_vm fi elif [ "$OP" = "delete" ]; then delete_backup +elif [ "$OP" = "stats" ]; then + get_backup_stats fi diff --git a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java index 3240bfcc8ab6..ed777486a122 100644 --- a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java +++ b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java @@ -37,11 +37,17 @@ import javax.mail.MessagingException; import javax.naming.ConfigurationException; +import com.cloud.dc.DataCenter; +import com.cloud.dc.Pod; +import com.cloud.org.Cluster; + +import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContextTimerTask; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.mailing.MailAddress; @@ -64,11 +70,9 @@ import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; import com.cloud.dc.ClusterVO; -import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenter.NetworkType; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; -import com.cloud.dc.Pod; import com.cloud.dc.Vlan.VlanType; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; @@ -82,7 +86,6 @@ import com.cloud.host.dao.HostDao; import com.cloud.network.Ipv6Service; import com.cloud.network.dao.IPAddressDao; -import com.cloud.org.Cluster; import com.cloud.org.Grouping.AllocationState; import com.cloud.resource.ResourceManager; import com.cloud.storage.StorageManager; @@ -143,8 +146,12 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi @Inject ConfigurationManager _configMgr; @Inject + protected BackupManager backupManager; + @Inject protected ConfigDepot _configDepot; @Inject + private ObjectStoreDao _objectStoreDao; + @Inject Ipv6Service ipv6Service; @Inject HostDao hostDao; @@ -157,6 +164,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi private double _vlanCapacityThreshold = 0.75; private double _directNetworkPublicIpCapacityThreshold = 0.75; private double _localStorageCapacityThreshold = 0.75; + private double _backupStorageCapacityThreshold = 0.75; + private double _objectStorageCapacityThreshold = 0.75; Map _capacityTypeThresholdMap = new HashMap<>(); private final ExecutorService _executor; @@ -199,6 +208,8 @@ public boolean configure(String name, Map params) throws Configu String vlanCapacityThreshold = _configDao.getValue(Config.VlanCapacityThreshold.key()); String directNetworkPublicIpCapacityThreshold = _configDao.getValue(Config.DirectNetworkPublicIpCapacityThreshold.key()); String localStorageCapacityThreshold = _configDao.getValue(Config.LocalStorageCapacityThreshold.key()); + String backupStorageCapacityThreshold = _configDao.getValue(BackupManager.BackupStorageCapacityThreshold.key()); + String objectStorageCapacityThreshold = _configDao.getValue(_storageMgr.ObjectStorageCapacityThreshold.key()); if (publicIPCapacityThreshold != null) { _publicIPCapacityThreshold = Double.parseDouble(publicIPCapacityThreshold); @@ -218,6 +229,12 @@ public boolean configure(String name, Map params) throws Configu if (localStorageCapacityThreshold != null) { _localStorageCapacityThreshold = Double.parseDouble(localStorageCapacityThreshold); } + if (backupStorageCapacityThreshold != null) { + _backupStorageCapacityThreshold = Double.parseDouble(backupStorageCapacityThreshold); + } + if (objectStorageCapacityThreshold != null) { + _objectStorageCapacityThreshold = Double.parseDouble(objectStorageCapacityThreshold); + } _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP, _publicIPCapacityThreshold); _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_PRIVATE_IP, _privateIPCapacityThreshold); @@ -226,6 +243,8 @@ public boolean configure(String name, Map params) throws Configu _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_DIRECT_ATTACHED_PUBLIC_IP, _directNetworkPublicIpCapacityThreshold); _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_LOCAL_STORAGE, _localStorageCapacityThreshold); _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET, Ipv6SubnetCapacityThreshold.value()); + _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_BACKUP_STORAGE, _backupStorageCapacityThreshold); + _capacityTypeThresholdMap.put(Capacity.CAPACITY_TYPE_OBJECT_STORAGE, _objectStorageCapacityThreshold); String capacityCheckPeriodStr = configs.get("capacity.check.period"); if (capacityCheckPeriodStr != null) { @@ -549,7 +568,9 @@ public void checkForAlerts() { for (Short capacityType : dataCenterCapacityTypes) { List capacity = _capacityDao.findCapacityBy(capacityType.intValue(), dc.getId(), null, null); - if (capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE) { + if (capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE || + capacityType == Capacity.CAPACITY_TYPE_OBJECT_STORAGE || + capacityType == Capacity.CAPACITY_TYPE_BACKUP_STORAGE) { capacity.add(getUsedStats(capacityType, dc.getId(), null, null)); } if (capacity == null || capacity.isEmpty()) { @@ -618,18 +639,22 @@ public void checkForAlerts() { } private SummedCapacity getUsedStats(short capacityType, long zoneId, Long podId, Long clusterId) { - CapacityVO capacity; + CapacityVO capacity = null; + if (capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE) { capacity = _storageMgr.getSecondaryStorageUsedStats(null, zoneId); - } else { + } else if (capacityType == Capacity.CAPACITY_TYPE_STORAGE) { capacity = _storageMgr.getStoragePoolUsedStats(null, clusterId, podId, zoneId); + } else if (capacityType == Capacity.CAPACITY_TYPE_OBJECT_STORAGE) { + capacity = _storageMgr.getObjectStorageUsedStats(zoneId); + } else if (capacityType == Capacity.CAPACITY_TYPE_BACKUP_STORAGE) { + capacity = (CapacityVO) backupManager.getBackupStorageUsedStats(zoneId); } if (capacity != null) { return new SummedCapacity(capacity.getUsedCapacity(), 0, capacity.getTotalCapacity(), capacityType, clusterId, podId); } else { return null; } - } private void generateEmailAlert(DataCenterVO dc, HostPodVO pod, ClusterVO cluster, double totalCapacity, double usedCapacity, short capacityType) { @@ -706,6 +731,16 @@ private void generateEmailAlert(DataCenterVO dc, HostPodVO pod, ClusterVO cluste msgContent = String.format("Number of unallocated virtual network guest IPv6 subnets is low, total: [%s], allocated: [%s] (%s%%).", totalInString, usedInString, percentual); alertType = AlertManager.AlertType.ALERT_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET; break; + case Capacity.CAPACITY_TYPE_BACKUP_STORAGE: + msgSubject = "System Alert: Low Available Backup Storage in availability zone " + dc.getName(); + msgContent = "Available backup storage space is low, total: " + totalInString + " MB, used: " + usedInString + " MB (" + percentual + "%)"; + alertType = AlertManager.AlertType.ALERT_TYPE_BACKUP_STORAGE; + break; + case Capacity.CAPACITY_TYPE_OBJECT_STORAGE: + msgSubject = "System Alert: Low Available Object Storage in availability zone " + dc.getName(); + msgContent = "Available object storage space is low, total: " + totalInString + " MB, used: " + usedInString + " MB (" + percentual + "%)"; + alertType = AlertManager.AlertType.ALERT_TYPE_OBJECT_STORAGE; + break; } try { @@ -724,6 +759,8 @@ private List getCapacityTypesAtZoneLevel() { dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_SECONDARY_STORAGE); dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_VLAN); dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET); + dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_BACKUP_STORAGE); + dataCenterCapacityTypes.add(Capacity.CAPACITY_TYPE_OBJECT_STORAGE); return dataCenterCapacityTypes; } diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index 1471ee4220a8..019d7ef213df 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -45,7 +45,6 @@ import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.AsyncJobResponse; import org.apache.cloudstack.api.response.BackupOfferingResponse; -import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.BackupScheduleResponse; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; @@ -75,7 +74,6 @@ import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.api.response.VpcOfferingResponse; import org.apache.cloudstack.api.response.ZoneResponse; -import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupOffering; import org.apache.cloudstack.backup.BackupSchedule; import org.apache.cloudstack.backup.dao.BackupDao; @@ -1065,6 +1063,10 @@ public static CapacityVO getSecondaryStorageUsedStats(Long hostId, Long zoneId) return s_storageMgr.getSecondaryStorageUsedStats(hostId, zoneId); } + public static CapacityVO getObjectStorageUsedStats(Long zoneId) { + return s_storageMgr.getObjectStorageUsedStats(zoneId); + } + // /////////////////////////////////////////////////////////// // Dao methods // // /////////////////////////////////////////////////////////// @@ -1102,7 +1104,7 @@ public static DiskOfferingVO findComputeOnlyDiskOfferingById(Long diskOfferingId return null; } - public static DiskOfferingVO findDiskOfferingById(Long diskOfferingId) { + public static DiskOfferingVO findNonComputeDiskOfferingById(Long diskOfferingId) { if (diskOfferingId == null) { return null; } @@ -1113,6 +1115,14 @@ public static DiskOfferingVO findDiskOfferingById(Long diskOfferingId) { return null; } + public static DiskOfferingVO findDiskOfferingById(Long diskOfferingId) { + if (diskOfferingId == null) { + return null; + } + DiskOfferingVO off = s_diskOfferingDao.findByIdIncludingRemoved(diskOfferingId); + return off; + } + public static ServiceOfferingVO findServiceOfferingByComputeOnlyDiskOffering(Long diskOfferingId, boolean includingRemoved) { ServiceOfferingVO off = s_serviceOfferingDao.findServiceOfferingByComputeOnlyDiskOffering(diskOfferingId, includingRemoved); return off; @@ -2264,10 +2274,6 @@ public static ResourceIconVO getResourceIconByResourceUUID(String resourceUUID, return s_resourceIconDao.findByResourceUuid(resourceUUID, resourceType); } - public static BackupResponse newBackupResponse(Backup backup) { - return s_backupDao.newBackupResponse(backup); - } - public static BackupScheduleResponse newBackupScheduleResponse(BackupSchedule schedule) { return s_backupScheduleDao.newBackupScheduleResponse(schedule); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 0293e0d08fbc..41433cb3e6f8 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -77,7 +77,6 @@ import org.apache.cloudstack.api.response.AutoScaleVmProfileResponse; import org.apache.cloudstack.api.response.BackupOfferingResponse; import org.apache.cloudstack.api.response.BackupRepositoryResponse; -import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.BackupScheduleResponse; import org.apache.cloudstack.api.response.BgpPeerResponse; import org.apache.cloudstack.api.response.BucketResponse; @@ -198,7 +197,6 @@ import org.apache.cloudstack.api.response.VpcResponse; import org.apache.cloudstack.api.response.VpnUsersResponse; import org.apache.cloudstack.api.response.ZoneResponse; -import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupOffering; import org.apache.cloudstack.backup.BackupRepository; import org.apache.cloudstack.backup.BackupSchedule; @@ -1483,6 +1481,7 @@ private static List getStatsCapacityresponse(Long poolId, Long capacities.add(ApiDBUtils.getStoragePoolUsedStats(poolId, clusterId, podId, zoneId)); if (clusterId == null && podId == null) { capacities.add(ApiDBUtils.getSecondaryStorageUsedStats(poolId, zoneId)); + capacities.add(ApiDBUtils.getObjectStorageUsedStats(zoneId)); } List capacityResponses = new ArrayList(); @@ -2140,6 +2139,11 @@ public List createCapacityResponse(List re List capacityResponses = new ArrayList(); for (Capacity summedCapacity : result) { + if (summedCapacity.getTotalCapacity() == 0 && + (summedCapacity.getCapacityType() == Capacity.CAPACITY_TYPE_BACKUP_STORAGE || + summedCapacity.getCapacityType() == Capacity.CAPACITY_TYPE_OBJECT_STORAGE)) { + continue; + } CapacityResponse capacityResponse = new CapacityResponse(); capacityResponse.setCapacityTotal(summedCapacity.getTotalCapacity()); if (summedCapacity.getAllocatedCapacity() != null) { @@ -5058,11 +5062,6 @@ public UserDataResponse createUserDataResponse(UserData userData) { return response; } - @Override - public BackupResponse createBackupResponse(Backup backup) { - return ApiDBUtils.newBackupResponse(backup); - } - @Override public BackupScheduleResponse createBackupScheduleResponse(BackupSchedule schedule) { return ApiDBUtils.newBackupScheduleResponse(schedule); diff --git a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java index 8092c63c6fd4..9762fc5ed3eb 100644 --- a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java @@ -253,7 +253,7 @@ public UserVmResponse newUserVmResponse(ResponseView view, String objectName, Us userVmResponse.setServiceOfferingName(userVm.getServiceOfferingName()); } if (details.contains(VMDetails.all) || details.contains(VMDetails.diskoff)) { - DiskOfferingVO diskOfferingVO = ApiDBUtils.findDiskOfferingById(userVm.getDiskOfferingId()); + DiskOfferingVO diskOfferingVO = ApiDBUtils.findNonComputeDiskOfferingById(userVm.getDiskOfferingId()); if (diskOfferingVO != null) { userVmResponse.setDiskOfferingId(userVm.getDiskOfferingUuid()); userVmResponse.setDiskOfferingName(userVm.getDiskOfferingName()); diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index 2554435a7877..8c86ed55df8f 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -6400,4 +6400,20 @@ public String getNicVlanValueForExternalVm(NicTO nic) { } return Networks.BroadcastDomainType.getValue(nic.getBroadcastUri()); } + + @Override + public Network.IpAddresses getIpAddressesFromIps(String ipAddress, String ip6Address, String macAddress) { + if (ip6Address != null) { + ip6Address = NetUtils.standardizeIp6Address(ip6Address); + } + if (macAddress != null) { + if (!NetUtils.isValidMac(macAddress)) { + throw new InvalidParameterValueException("Mac address is not valid: " + macAddress); + } else if (!NetUtils.isUnicastMac(macAddress)) { + throw new InvalidParameterValueException("Mac address is not unicast: " + macAddress); + } + macAddress = NetUtils.standardizeMacAddress(macAddress); + } + return new Network.IpAddresses(ipAddress, ip6Address, macAddress); + } } diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 7c7e0b662d73..83493ad702b7 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -1816,7 +1816,7 @@ protected UserVm createNewVM(AutoScaleVmGroupVO asGroup) { if (zone.getNetworkType() == NetworkType.Basic) { vm = userVmService.createBasicSecurityGroupVirtualMachine(zone, serviceOffering, template, null, owner, vmHostName, - vmHostName, diskOfferingId, dataDiskSize, null, + vmHostName, diskOfferingId, dataDiskSize, null, null, hypervisorType, HTTPMethod.GET, userData, userDataId, userDataDetails, sshKeyPairs, null, null, true, null, affinityGroupIdList, customParameters, null, null, null, null, true, overrideDiskOfferingId, null, null); @@ -1824,13 +1824,13 @@ protected UserVm createNewVM(AutoScaleVmGroupVO asGroup) { if (networkModel.checkSecurityGroupSupportForNetwork(owner, zone, networkIds, Collections.emptyList())) { vm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, template, networkIds, null, - owner, vmHostName,vmHostName, diskOfferingId, dataDiskSize, null, + owner, vmHostName,vmHostName, diskOfferingId, dataDiskSize, null, null, hypervisorType, HTTPMethod.GET, userData, userDataId, userDataDetails, sshKeyPairs, null, null, true, null, affinityGroupIdList, customParameters, null, null, null, null, true, overrideDiskOfferingId, null, null, null); } else { vm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, vmHostName, vmHostName, - diskOfferingId, dataDiskSize, null, + diskOfferingId, dataDiskSize, null, null, hypervisorType, HTTPMethod.GET, userData, userDataId, userDataDetails, sshKeyPairs, null, addrs, true, null, affinityGroupIdList, customParameters, null, null, null, null, true, null, overrideDiskOfferingId, null, null); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index b09a675aa588..4cd16b42ae94 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -631,6 +631,7 @@ import org.apache.cloudstack.api.command.user.zone.ListZonesCmd; import org.apache.cloudstack.auth.UserAuthenticator; import org.apache.cloudstack.auth.UserTwoFactorAuthenticator; +import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.config.ConfigurationGroup; @@ -1045,6 +1046,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @Inject StoragePoolTagsDao storagePoolTagsDao; @Inject + private BackupManager backupManager; + @Inject protected ManagementServerJoinDao managementServerJoinDao; @Inject ClusterManager _clusterMgr; @@ -3560,7 +3563,8 @@ public List listTopConsumedResources(final ListCapacityCmd cmd) { } List getStorageCapacities(Long clusterId, Long podId, Long zoneId, List poolIds, Short capacityType) { - List capacityTypes = Arrays.asList(Capacity.CAPACITY_TYPE_STORAGE, Capacity.CAPACITY_TYPE_SECONDARY_STORAGE); + List capacityTypes = Arrays.asList(Capacity.CAPACITY_TYPE_STORAGE, Capacity.CAPACITY_TYPE_SECONDARY_STORAGE, + Capacity.CAPACITY_TYPE_BACKUP_STORAGE, Capacity.CAPACITY_TYPE_OBJECT_STORAGE); if (capacityType != null && !capacityTypes.contains(capacityType)) { return null; } @@ -3568,7 +3572,7 @@ List getStorageCapacities(Long clusterId, Long podId, Long zoneI capacityTypes = capacityTypes.stream().filter(x -> x.equals(capacityType)).collect(Collectors.toList()); } if (CollectionUtils.isNotEmpty(poolIds)) { - capacityTypes = capacityTypes.stream().filter(x -> x != Capacity.CAPACITY_TYPE_SECONDARY_STORAGE).collect(Collectors.toList()); + capacityTypes = capacityTypes.stream().filter(x -> x == Capacity.CAPACITY_TYPE_STORAGE).collect(Collectors.toList()); } if (CollectionUtils.isEmpty(capacityTypes)) { return null; @@ -3594,6 +3598,12 @@ List getStorageCapacities(Long clusterId, Long podId, Long zoneI if (capacityTypes.contains(Capacity.CAPACITY_TYPE_STORAGE)) { capacities.add(_storageMgr.getStoragePoolUsedStats(dc.getId(), podId, clusterId, poolIds)); } + if (capacityTypes.contains(Capacity.CAPACITY_TYPE_OBJECT_STORAGE)) { + capacities.add(_storageMgr.getObjectStorageUsedStats(dc.getId())); + } + if (capacityTypes.contains(Capacity.CAPACITY_TYPE_BACKUP_STORAGE)) { + capacities.add((CapacityVO) backupManager.getBackupStorageUsedStats(dc.getId())); + } for (CapacityVO capacity : capacities) { if (capacity.getTotalCapacity() != 0) { capacity.setUsedPercentage((float)capacity.getUsedCapacity() / capacity.getTotalCapacity()); @@ -3608,6 +3618,22 @@ List getStorageCapacities(Long clusterId, Long podId, Long zoneI return list; } + private void addZoneWideCapacitiesByType(final Integer capacityType, Long zId, List taggedCapacities) { + if (capacityType == null) { + taggedCapacities.add(_storageMgr.getSecondaryStorageUsedStats(null, zId)); + taggedCapacities.add(_storageMgr.getObjectStorageUsedStats(zId)); + taggedCapacities.add((CapacityVO) backupManager.getBackupStorageUsedStats(zId)); + return; + } + + if (capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE) { + taggedCapacities.add(_storageMgr.getSecondaryStorageUsedStats(null, zId)); + } else if (capacityType == Capacity.CAPACITY_TYPE_OBJECT_STORAGE) { + taggedCapacities.add(_storageMgr.getObjectStorageUsedStats(zId)); + } else if (capacityType == Capacity.CAPACITY_TYPE_BACKUP_STORAGE) { + taggedCapacities.add((CapacityVO) backupManager.getBackupStorageUsedStats(zId)); + } + } protected List listCapacitiesWithDetails(final Long zoneId, final Long podId, Long clusterId, final Integer capacityType, final String tag, List dcList) { @@ -3636,11 +3662,9 @@ protected List listCapacitiesWithDetails(final Long zoneId, final Lo for (final Long zId : dcList) { // op_host_Capacity contains only allocated stats and the real time // stats are stored "in memory". - // List secondary storage capacity only when the api is invoked for the zone layer. - if ((capacityType == null || capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE) && - podId == null && clusterId == null && - StringUtils.isEmpty(t)) { - taggedCapacities.add(_storageMgr.getSecondaryStorageUsedStats(null, zId)); + // List secondary, object and backup storage capacities only when the api is invoked for the zone layer. + if (podId == null && clusterId == null && StringUtils.isEmpty(t)) { + addZoneWideCapacitiesByType(capacityType, zId, taggedCapacities); } if ((capacityType == null || capacityType == Capacity.CAPACITY_TYPE_STORAGE) && storagePoolIdsForCapacity.first()) { taggedCapacities.add(_storageMgr.getStoragePoolUsedStats(zId, podId, clusterId, storagePoolIdsForCapacity.second())); diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 76be0ed6b56c..f6bef8b2e8cf 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -4588,7 +4588,8 @@ public ConfigKey[] getConfigKeys() { VmwareAllowParallelExecution, DataStoreDownloadFollowRedirects, AllowVolumeReSizeBeyondAllocation, - StoragePoolHostConnectWorkers + StoragePoolHostConnectWorkers, + ObjectStorageCapacityThreshold }; } @@ -4637,7 +4638,7 @@ private void setVolumeObjectTOThrottling(VolumeObjectTO volumeTO, final ServiceO @Override @ActionEvent(eventType = EventTypes.EVENT_OBJECT_STORE_CREATE, eventDescription = "creating object storage") - public ObjectStore discoverObjectStore(String name, String url, String providerName, Map details) + public ObjectStore discoverObjectStore(String name, String url, Long size, String providerName, Map details) throws IllegalArgumentException, InvalidParameterValueException { DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(providerName); @@ -4667,6 +4668,11 @@ public ObjectStore discoverObjectStore(String name, String url, String providerN Map params = new HashMap<>(); params.put("url", url); params.put("name", name); + if (size == null) { + params.put("size", 0L); + } else { + params.put("size", size); + } params.put("providerName", storeProvider.getName()); params.put("role", DataStoreRole.Object); params.put("details", details); @@ -4750,8 +4756,28 @@ public ObjectStore updateObjectStore(Long id, UpdateObjectStoragePoolCmd cmd) { if(cmd.getName() != null ) { objectStoreVO.setName(cmd.getName()); } + if (cmd.getSize() != null) { + objectStoreVO.setTotalSize(cmd.getSize() * ResourceType.bytesToGiB); + } _objectStoreDao.update(id, objectStoreVO); logger.debug("Successfully updated object store: {}", objectStoreVO); return objectStoreVO; } + + @Override + public CapacityVO getObjectStorageUsedStats(Long zoneId) { + List objectStores = _objectStoreDao.listObjectStores(); + Long allocated = 0L; + Long total = 0L; + for (ObjectStoreVO objectStore: objectStores) { + if (objectStore.getAllocatedSize() != null) { + allocated += objectStore.getAllocatedSize(); + } + if (objectStore.getTotalSize() != null) { + total += objectStore.getTotalSize(); + } + } + CapacityVO capacity = new CapacityVO(null, zoneId, null, null, allocated, total, Capacity.CAPACITY_TYPE_OBJECT_STORAGE); + return capacity; + } } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 8f374029b1d6..529ed3f1d7b8 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -954,7 +954,7 @@ public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationExcept String userSpecifiedName = getVolumeNameFromCommand(cmd); - return commitVolume(cmd, caller, owner, displayVolume, zoneId, diskOfferingId, provisioningType, size, minIops, maxIops, parentVolume, userSpecifiedName, + return commitVolume(cmd.getSnapshotId(), caller, owner, displayVolume, zoneId, diskOfferingId, provisioningType, size, minIops, maxIops, parentVolume, userSpecifiedName, _uuidMgr.generateUuid(Volume.class, cmd.getCustomId()), details); } @@ -968,7 +968,7 @@ public void validateCustomDiskOfferingSizeRange(Long sizeInGB) { } } - private VolumeVO commitVolume(final CreateVolumeCmd cmd, final Account caller, final Account owner, final Boolean displayVolume, final Long zoneId, final Long diskOfferingId, + private VolumeVO commitVolume(final Long snapshotId, final Account caller, final Account owner, final Boolean displayVolume, final Long zoneId, final Long diskOfferingId, final Storage.ProvisioningType provisioningType, final Long size, final Long minIops, final Long maxIops, final VolumeVO parentVolume, final String userSpecifiedName, final String uuid, final Map details) { return Transaction.execute(new TransactionCallback() { @Override @@ -996,7 +996,7 @@ public VolumeVO doInTransaction(TransactionStatus status) { volume = _volsDao.persist(volume); - if (cmd.getSnapshotId() == null && displayVolume) { + if (snapshotId == null && displayVolume) { // for volume created from snapshot, create usage event after volume creation UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), diskOfferingId, null, size, Volume.class.getName(), volume.getUuid(), displayVolume); @@ -2867,7 +2867,10 @@ protected String createVolumeInfoFromVolumes(List vmVolumes) { try { List list = new ArrayList<>(); for (VolumeVO vol : vmVolumes) { - list.add(new Backup.VolumeInfo(vol.getUuid(), vol.getPath(), vol.getVolumeType(), vol.getSize())); + DiskOfferingVO diskOffering = _diskOfferingDao.findById(vol.getDiskOfferingId()); + String diskOfferingUuid = diskOffering != null ? diskOffering.getUuid() : null; + list.add(new Backup.VolumeInfo(vol.getUuid(), vol.getPath(), vol.getVolumeType(), vol.getSize(), + vol.getDeviceId(), diskOfferingUuid, vol.getMinIops(), vol.getMaxIops())); } return GsonHelper.getGson().toJson(list.toArray(), Backup.VolumeInfo[].class); } catch (Exception e) { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 91897c0977de..6cdcba7e3b62 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.StringReader; import java.io.UnsupportedEncodingException; +import java.lang.reflect.Type; import java.net.URLDecoder; import java.text.SimpleDateFormat; import java.time.LocalDateTime; @@ -74,10 +75,13 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.BaseCmd.HTTPMethod; import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd; +import org.apache.cloudstack.api.command.admin.vm.CreateVMFromBackupCmdByAdmin; import org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin; import org.apache.cloudstack.api.command.admin.vm.ExpungeVMCmd; import org.apache.cloudstack.api.command.admin.vm.RecoverVMCmd; import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; +import org.apache.cloudstack.api.command.user.vm.BaseDeployVMCmd; +import org.apache.cloudstack.api.command.user.vm.CreateVMFromBackupCmd; import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; import org.apache.cloudstack.api.command.user.vm.DeployVnfApplianceCmd; import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; @@ -98,8 +102,8 @@ import org.apache.cloudstack.api.command.user.vmgroup.DeleteVMGroupCmd; import org.apache.cloudstack.api.command.user.volume.ChangeOfferingForVolumeCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; -import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.backup.BackupVO; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.cloud.entity.api.VirtualMachineEntity; @@ -399,6 +403,8 @@ import com.cloud.vm.snapshot.VMSnapshotManager; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; public class UserVmManagerImpl extends ManagerBase implements UserVmManager, VirtualMachineGuru, Configurable { @@ -1008,14 +1014,12 @@ public UserVm resetVMUserData(ResetVMUserDataCmd cmd) throws ResourceUnavailable @Override @ActionEvent(eventType = EventTypes.EVENT_VM_RESETSSHKEY, eventDescription = "resetting Vm SSHKey", async = true) public UserVm resetVMSSHKey(ResetVMSSHKeyCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException { - Account caller = CallContext.current().getCallingAccount(); Account owner = _accountMgr.finalizeOwner(caller, cmd.getAccountName(), cmd.getDomainId(), cmd.getProjectId()); - Long vmId = cmd.getId(); UserVmVO userVm = _vmDao.findById(cmd.getId()); if (userVm == null) { - throw new InvalidParameterValueException("unable to find a virtual machine by id" + cmd.getId()); + throw new InvalidParameterValueException("unable to find a virtual machine by id " + cmd.getId()); } if (UserVmManager.SHAREDFSVM.equals(userVm.getUserVmType())) { throw new InvalidParameterValueException("Operation not supported on Shared FileSystem Instance"); @@ -1027,10 +1031,7 @@ public UserVm resetVMSSHKey(ResetVMSSHKeyCmd cmd) throws ResourceUnavailableExce userVm.getName())); } - VMTemplateVO template = _templateDao.findByIdIncludingRemoved(userVm.getTemplateId()); - // Do parameters input validation - if (userVm.getState() == State.Error || userVm.getState() == State.Expunging) { logger.error("vm ({}) is not in the right state: {}", userVm, userVm.getState()); throw new InvalidParameterValueException("Vm with specified id is not in the right state"); @@ -1040,32 +1041,40 @@ public UserVm resetVMSSHKey(ResetVMSSHKeyCmd cmd) throws ResourceUnavailableExce throw new InvalidParameterValueException("Vm " + userVm + " should be stopped to do SSH Key reset"); } - if (cmd.getNames() == null || cmd.getNames().isEmpty()) { + List names = cmd.getNames(); + if (CollectionUtils.isEmpty(names)) { throw new InvalidParameterValueException("'keypair' or 'keypairs' must be specified"); } + userVm = resetVMSSHKeyInternal(userVm, owner, names); + return userVm; + } + + private UserVmVO resetVMSSHKeyInternal(UserVmVO userVm, Account owner, List names) throws ResourceUnavailableException, InsufficientCapacityException { + Account caller = CallContext.current().getCallingAccount(); + String keypairnames = ""; String sshPublicKeys = ""; List pairs = new ArrayList<>(); - pairs = _sshKeyPairDao.findByNames(owner.getAccountId(), owner.getDomainId(), cmd.getNames()); - if (pairs == null || pairs.size() != cmd.getNames().size()) { + pairs = _sshKeyPairDao.findByNames(owner.getAccountId(), owner.getDomainId(), names); + if (pairs == null || pairs.size() != names.size()) { throw new InvalidParameterValueException("Not all specified keypairs exist"); } sshPublicKeys = pairs.stream().map(p -> p.getPublicKey()).collect(Collectors.joining("\n")); - keypairnames = String.join(",", cmd.getNames()); + keypairnames = String.join(",", names); _accountMgr.checkAccess(caller, null, true, userVm); - boolean result = resetVMSSHKeyInternal(vmId, sshPublicKeys, keypairnames); + boolean result = resetVMSSHKeyInternal(userVm.getId(), sshPublicKeys, keypairnames); - UserVmVO vm = _vmDao.findById(vmId); + UserVmVO vm = _vmDao.findById(userVm.getId()); _vmDao.loadDetails(vm); if (!result) { throw new CloudRuntimeException("Failed to reset SSH Key for the virtual machine "); } - removeEncryptedPasswordFromUserVmVoDetails(vmId); + removeEncryptedPasswordFromUserVmVoDetails(userVm.getId()); _vmDao.loadDetails(userVm); return userVm; @@ -2515,17 +2524,7 @@ public boolean expunge(UserVmVO vm) { } try { - if (vm.getBackupOfferingId() != null) { - List backupsForVm = backupDao.listByVmId(vm.getDataCenterId(), vm.getId()); - if (CollectionUtils.isEmpty(backupsForVm)) { - backupManager.removeVMFromBackupOffering(vm.getId(), true); - } else { - throw new CloudRuntimeException(String.format("This VM [uuid: %s, name: %s] has a " - + "Backup Offering [id: %s, external id: %s] with %s backups. Please, remove the backup offering " - + "before proceeding to VM exclusion!", vm.getUuid(), vm.getInstanceName(), vm.getBackupOfferingId(), - vm.getBackupExternalId(), backupsForVm.size())); - } - } + backupManager.checkAndRemoveBackupOfferingBeforeExpunge(vm); autoScaleManager.removeVmFromVmGroup(vm.getId()); @@ -3523,6 +3522,10 @@ public UserVm destroyVm(DestroyVMCmd cmd) throws ResourceUnavailableException, C final ControlledEntity[] volumesToDelete = volumesToBeDeleted.toArray(new ControlledEntity[0]); _accountMgr.checkAccess(ctx.getCallingAccount(), null, true, volumesToDelete); + if (expunge) { + backupManager.checkAndRemoveBackupOfferingBeforeExpunge(vm); + } + stopVirtualMachine(vmId, VmDestroyForcestop.value()); // Detach all data disks from VM @@ -3752,7 +3755,7 @@ private boolean validPassword(String password) { @Override @ActionEvent(eventType = EventTypes.EVENT_VM_CREATE, eventDescription = "deploying Vm", create = true) public UserVm createBasicSecurityGroupVirtualMachine(DataCenter zone, ServiceOffering serviceOffering, VirtualMachineTemplate template, List securityGroupIdList, - Account owner, String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor, HTTPMethod httpmethod, + Account owner, String hostName, String displayName, Long diskOfferingId, Long diskSize, List dataDiskInfoList, String group, HypervisorType hypervisor, HTTPMethod httpmethod, String userData, Long userDataId, String userDataDetails, List sshKeyPairs, Map requestedIps, IpAddresses defaultIps, Boolean displayVm, String keyboard, List affinityGroupIdList, Map customParametes, String customId, Map> dhcpOptionMap, Map dataDiskTemplateToDiskOfferingMap, Map userVmOVFProperties, boolean dynamicScalingEnabled, Long overrideDiskOfferingId, Volume volume, Snapshot snapshot) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException, @@ -3802,7 +3805,7 @@ public UserVm createBasicSecurityGroupVirtualMachine(DataCenter zone, ServiceOff } } - return createVirtualMachine(zone, serviceOffering, template, hostName, displayName, owner, diskOfferingId, diskSize, networkList, securityGroupIdList, group, httpmethod, + return createVirtualMachine(zone, serviceOffering, template, hostName, displayName, owner, diskOfferingId, diskSize, dataDiskInfoList, networkList, securityGroupIdList, group, httpmethod, userData, userDataId, userDataDetails, sshKeyPairs, hypervisor, caller, requestedIps, defaultIps, displayVm, keyboard, affinityGroupIdList, customParametes, customId, dhcpOptionMap, dataDiskTemplateToDiskOfferingMap, userVmOVFProperties, dynamicScalingEnabled, null, overrideDiskOfferingId, volume, snapshot); @@ -3811,7 +3814,7 @@ public UserVm createBasicSecurityGroupVirtualMachine(DataCenter zone, ServiceOff @Override @ActionEvent(eventType = EventTypes.EVENT_VM_CREATE, eventDescription = "deploying Vm", create = true) public UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, ServiceOffering serviceOffering, VirtualMachineTemplate template, List networkIdList, - List securityGroupIdList, Account owner, String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor, + List securityGroupIdList, Account owner, String hostName, String displayName, Long diskOfferingId, Long diskSize, List dataDiskInfoList, String group, HypervisorType hypervisor, HTTPMethod httpmethod, String userData, Long userDataId, String userDataDetails, List sshKeyPairs, Map requestedIps, IpAddresses defaultIps, Boolean displayVm, String keyboard, List affinityGroupIdList, Map customParameters, String customId, Map> dhcpOptionMap, Map dataDiskTemplateToDiskOfferingMap, Map userVmOVFProperties, boolean dynamicScalingEnabled, Long overrideDiskOfferingId, String vmType, Volume volume, Snapshot snapshot) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException, StorageUnavailableException, ResourceAllocationException { @@ -3915,7 +3918,7 @@ public UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, Service } } - return createVirtualMachine(zone, serviceOffering, template, hostName, displayName, owner, diskOfferingId, diskSize, networkList, securityGroupIdList, group, httpmethod, + return createVirtualMachine(zone, serviceOffering, template, hostName, displayName, owner, diskOfferingId, diskSize, dataDiskInfoList, networkList, securityGroupIdList, group, httpmethod, userData, userDataId, userDataDetails, sshKeyPairs, hypervisor, caller, requestedIps, defaultIps, displayVm, keyboard, affinityGroupIdList, customParameters, customId, dhcpOptionMap, dataDiskTemplateToDiskOfferingMap, userVmOVFProperties, dynamicScalingEnabled, vmType, overrideDiskOfferingId, volume, snapshot); } @@ -3923,7 +3926,7 @@ public UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, Service @Override @ActionEvent(eventType = EventTypes.EVENT_VM_CREATE, eventDescription = "deploying Vm", create = true) public UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serviceOffering, VirtualMachineTemplate template, List networkIdList, Account owner, - String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor, HTTPMethod httpmethod, String userData, + String hostName, String displayName, Long diskOfferingId, Long diskSize, List dataDiskInfoList, String group, HypervisorType hypervisor, HTTPMethod httpmethod, String userData, Long userDataId, String userDataDetails, List sshKeyPairs, Map requestedIps, IpAddresses defaultIps, Boolean displayvm, String keyboard, List affinityGroupIdList, Map customParametrs, String customId, Map> dhcpOptionsMap, Map dataDiskTemplateToDiskOfferingMap, Map userVmOVFPropertiesMap, boolean dynamicScalingEnabled, String vmType, Long overrideDiskOfferingId, Volume volume, Snapshot snapshot) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException, @@ -3977,7 +3980,7 @@ public UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serv } } verifyExtraDhcpOptionsNetwork(dhcpOptionsMap, networkList); - return createVirtualMachine(zone, serviceOffering, template, hostName, displayName, owner, diskOfferingId, diskSize, networkList, null, group, httpmethod, userData, + return createVirtualMachine(zone, serviceOffering, template, hostName, displayName, owner, diskOfferingId, diskSize, dataDiskInfoList, networkList, null, group, httpmethod, userData, userDataId, userDataDetails, sshKeyPairs, hypervisor, caller, requestedIps, defaultIps, displayvm, keyboard, affinityGroupIdList, customParametrs, customId, dhcpOptionsMap, dataDiskTemplateToDiskOfferingMap, userVmOVFPropertiesMap, dynamicScalingEnabled, vmType, overrideDiskOfferingId, volume, snapshot); } @@ -4107,7 +4110,7 @@ public void checkNameForRFCCompliance(String name) { @DB private UserVm createVirtualMachine(DataCenter zone, ServiceOffering serviceOffering, VirtualMachineTemplate tmplt, String hostName, String displayName, Account owner, - Long diskOfferingId, Long diskSize, List networkList, List securityGroupIdList, String group, HTTPMethod httpmethod, String userData, + Long diskOfferingId, Long diskSize, List dataDiskInfoList, List networkList, List securityGroupIdList, String group, HTTPMethod httpmethod, String userData, Long userDataId, String userDataDetails, List sshKeyPairs, HypervisorType hypervisor, Account caller, Map requestedIps, IpAddresses defaultIps, Boolean isDisplayVm, String keyboard, List affinityGroupIdList, Map customParameters, String customId, Map> dhcpOptionMap, Map datadiskTemplateToDiskOfferringMap, @@ -4206,12 +4209,7 @@ private UserVm createVirtualMachine(DataCenter zone, ServiceOffering serviceOffe throw new InvalidParameterValueException("Root volume encryption is not supported for hypervisor type " + hypervisorType); } - long additionalDiskSize = 0L; - if (!isIso && diskOfferingId != null) { - DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId); - additionalDiskSize = verifyAndGetDiskSize(diskOffering, diskSize); - } - UserVm vm = getCheckedUserVmResource(zone, hostName, displayName, owner, diskOfferingId, diskSize, networkList, securityGroupIdList, group, httpmethod, userData, userDataId, userDataDetails, sshKeyPairs, caller, requestedIps, defaultIps, isDisplayVm, keyboard, affinityGroupIdList, customParameters, customId, dhcpOptionMap, datadiskTemplateToDiskOfferringMap, userVmOVFPropertiesMap, dynamicScalingEnabled, vmType, template, hypervisorType, accountId, offering, isIso, rootDiskOfferingId, volumesSize, additionalDiskSize, volume, snapshot); + UserVm vm = getCheckedUserVmResource(zone, hostName, displayName, owner, diskOfferingId, diskSize, dataDiskInfoList, networkList, securityGroupIdList, group, httpmethod, userData, userDataId, userDataDetails, sshKeyPairs, caller, requestedIps, defaultIps, isDisplayVm, keyboard, affinityGroupIdList, customParameters, customId, dhcpOptionMap, datadiskTemplateToDiskOfferringMap, userVmOVFPropertiesMap, dynamicScalingEnabled, vmType, template, hypervisorType, accountId, offering, isIso, rootDiskOfferingId, volumesSize, volume, snapshot); _securityGroupMgr.addInstanceToGroups(vm, securityGroupIdList); @@ -4224,14 +4222,14 @@ private UserVm createVirtualMachine(DataCenter zone, ServiceOffering serviceOffe } private UserVm getCheckedUserVmResource(DataCenter zone, String hostName, String displayName, Account owner, - Long diskOfferingId, Long diskSize, List networkList, List securityGroupIdList, String group, + Long diskOfferingId, Long diskSize, List dataDiskInfoList, List networkList, List securityGroupIdList, String group, HTTPMethod httpmethod, String userData, Long userDataId, String userDataDetails, List sshKeyPairs, Account caller, Map requestedIps, IpAddresses defaultIps, Boolean isDisplayVm, String keyboard, List affinityGroupIdList, Map customParameters, String customId, Map> dhcpOptionMap, Map datadiskTemplateToDiskOfferringMap, Map userVmOVFPropertiesMap, boolean dynamicScalingEnabled, String vmType, VMTemplateVO template, HypervisorType hypervisorType, long accountId, ServiceOfferingVO offering, boolean isIso, - Long rootDiskOfferingId, long volumesSize, long additionalDiskSize, Volume volume, Snapshot snapshot) throws ResourceAllocationException { + Long rootDiskOfferingId, long volumesSize, Volume volume, Snapshot snapshot) throws ResourceAllocationException { if (!VirtualMachineManager.ResourceCountRunningVMsonly.value()) { List resourceLimitHostTags = resourceLimitService.getResourceLimitHostTags(offering, template); try (CheckedReservation vmReservation = new CheckedReservation(owner, ResourceType.user_vm, resourceLimitHostTags, 1l, reservationDao, resourceLimitService); @@ -4240,7 +4238,7 @@ private UserVm getCheckedUserVmResource(DataCenter zone, String hostName, String CheckedReservation gpuReservation = offering.getGpuCount() != null && offering.getGpuCount() > 0 ? new CheckedReservation(owner, ResourceType.gpu, resourceLimitHostTags, Long.valueOf(offering.getGpuCount()), reservationDao, resourceLimitService) : null; ) { - return getUncheckedUserVmResource(zone, hostName, displayName, owner, diskOfferingId, diskSize, networkList, securityGroupIdList, group, httpmethod, userData, userDataId, userDataDetails, sshKeyPairs, caller, requestedIps, defaultIps, isDisplayVm, keyboard, affinityGroupIdList, customParameters, customId, dhcpOptionMap, datadiskTemplateToDiskOfferringMap, userVmOVFPropertiesMap, dynamicScalingEnabled, vmType, template, hypervisorType, accountId, offering, isIso, rootDiskOfferingId, volumesSize, additionalDiskSize, volume, snapshot); + return getUncheckedUserVmResource(zone, hostName, displayName, owner, diskOfferingId, diskSize, dataDiskInfoList, networkList, securityGroupIdList, group, httpmethod, userData, userDataId, userDataDetails, sshKeyPairs, caller, requestedIps, defaultIps, isDisplayVm, keyboard, affinityGroupIdList, customParameters, customId, dhcpOptionMap, datadiskTemplateToDiskOfferringMap, userVmOVFPropertiesMap, dynamicScalingEnabled, vmType, template, hypervisorType, accountId, offering, isIso, rootDiskOfferingId, volumesSize, volume, snapshot); } catch (ResourceAllocationException | CloudRuntimeException e) { throw e; } catch (Exception e) { @@ -4249,7 +4247,7 @@ private UserVm getCheckedUserVmResource(DataCenter zone, String hostName, String } } else { - return getUncheckedUserVmResource(zone, hostName, displayName, owner, diskOfferingId, diskSize, networkList, securityGroupIdList, group, httpmethod, userData, userDataId, userDataDetails, sshKeyPairs, caller, requestedIps, defaultIps, isDisplayVm, keyboard, affinityGroupIdList, customParameters, customId, dhcpOptionMap, datadiskTemplateToDiskOfferringMap, userVmOVFPropertiesMap, dynamicScalingEnabled, vmType, template, hypervisorType, accountId, offering, isIso, rootDiskOfferingId, volumesSize, additionalDiskSize, volume, snapshot); + return getUncheckedUserVmResource(zone, hostName, displayName, owner, diskOfferingId, diskSize, dataDiskInfoList, networkList, securityGroupIdList, group, httpmethod, userData, userDataId, userDataDetails, sshKeyPairs, caller, requestedIps, defaultIps, isDisplayVm, keyboard, affinityGroupIdList, customParameters, customId, dhcpOptionMap, datadiskTemplateToDiskOfferringMap, userVmOVFPropertiesMap, dynamicScalingEnabled, vmType, template, hypervisorType, accountId, offering, isIso, rootDiskOfferingId, volumesSize, volume, snapshot); } } @@ -4258,24 +4256,53 @@ protected List getResourceLimitStorageTags(long diskOfferingId) { return resourceLimitService.getResourceLimitStorageTags(diskOfferingVO); } + private List reserveStorageResourcesForVm(Account owner, Long diskOfferingId, Long diskSize, List dataDiskInfoList, Long rootDiskOfferingId, ServiceOfferingVO offering, Long rootDiskSize) throws ResourceAllocationException { + List checkedReservations = new ArrayList<>(); + + List rootResourceLimitStorageTags = getResourceLimitStorageTags(rootDiskOfferingId != null ? rootDiskOfferingId : offering.getDiskOfferingId()); + CheckedReservation rootVolumeReservation = new CheckedReservation(owner, ResourceType.volume, rootResourceLimitStorageTags, 1L, reservationDao, resourceLimitService); + checkedReservations.add(rootVolumeReservation); + CheckedReservation rootPrimaryStorageReservation = new CheckedReservation(owner, ResourceType.primary_storage, rootResourceLimitStorageTags, rootDiskSize, reservationDao, resourceLimitService); + checkedReservations.add(rootPrimaryStorageReservation); + + if (diskOfferingId != null) { + List additionalResourceLimitStorageTags = diskOfferingId != null ? getResourceLimitStorageTags(diskOfferingId) : null; + DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId); + Long size = verifyAndGetDiskSize(diskOffering, diskSize); + CheckedReservation additionalVolumeReservation = diskOfferingId != null ? new CheckedReservation(owner, ResourceType.volume, additionalResourceLimitStorageTags, 1L, reservationDao, resourceLimitService) : null; + checkedReservations.add(additionalVolumeReservation); + CheckedReservation additionalPrimaryStorageReservation = diskOfferingId != null ? new CheckedReservation(owner, ResourceType.primary_storage, additionalResourceLimitStorageTags, size, reservationDao, resourceLimitService) : null; + checkedReservations.add(additionalPrimaryStorageReservation); + + } + + if (dataDiskInfoList != null) { + for (VmDiskInfo vmDiskInfo : dataDiskInfoList) { + DiskOffering diskOffering = vmDiskInfo.getDiskOffering(); + List additionalResourceLimitStorageTagsForDataDisk = getResourceLimitStorageTags(vmDiskInfo.getDiskOffering().getId()); + Long size = verifyAndGetDiskSize(diskOffering, vmDiskInfo.getSize()); + CheckedReservation additionalVolumeReservation = new CheckedReservation(owner, ResourceType.volume, additionalResourceLimitStorageTagsForDataDisk, 1L, reservationDao, resourceLimitService); + checkedReservations.add(additionalVolumeReservation); + CheckedReservation additionalPrimaryStorageReservation = new CheckedReservation(owner, ResourceType.primary_storage, additionalResourceLimitStorageTagsForDataDisk, size, reservationDao, resourceLimitService); + checkedReservations.add(additionalPrimaryStorageReservation); + } + } + return checkedReservations; + } + private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, String displayName, Account owner, - Long diskOfferingId, Long diskSize, List networkList, List securityGroupIdList, String group, + Long diskOfferingId, Long diskSize, List dataDiskInfoList, List networkList, List securityGroupIdList, String group, HTTPMethod httpmethod, String userData, Long userDataId, String userDataDetails, List sshKeyPairs, Account caller, Map requestedIps, IpAddresses defaultIps, Boolean isDisplayVm, String keyboard, List affinityGroupIdList, Map customParameters, String customId, Map> dhcpOptionMap, Map datadiskTemplateToDiskOfferringMap, Map userVmOVFPropertiesMap, boolean dynamicScalingEnabled, String vmType, VMTemplateVO template, HypervisorType hypervisorType, long accountId, ServiceOfferingVO offering, boolean isIso, - Long rootDiskOfferingId, long volumesSize, long additionalDiskSize, Volume volume, Snapshot snapshot) throws ResourceAllocationException - { - List rootResourceLimitStorageTags = getResourceLimitStorageTags(rootDiskOfferingId != null ? rootDiskOfferingId : offering.getDiskOfferingId()); - List additionalResourceLimitStorageTags = diskOfferingId != null ? getResourceLimitStorageTags(diskOfferingId) : null; + Long rootDiskOfferingId, long volumesSize, Volume volume, Snapshot snapshot) throws ResourceAllocationException { + List checkedReservations = new ArrayList<>(); - try (CheckedReservation rootVolumeReservation = new CheckedReservation(owner, ResourceType.volume, rootResourceLimitStorageTags, 1L, reservationDao, resourceLimitService); - CheckedReservation additionalVolumeReservation = diskOfferingId != null ? new CheckedReservation(owner, ResourceType.volume, additionalResourceLimitStorageTags, 1L, reservationDao, resourceLimitService) : null; - CheckedReservation rootPrimaryStorageReservation = new CheckedReservation(owner, ResourceType.primary_storage, rootResourceLimitStorageTags, volumesSize, reservationDao, resourceLimitService); - CheckedReservation additionalPrimaryStorageReservation = diskOfferingId != null ? new CheckedReservation(owner, ResourceType.primary_storage, additionalResourceLimitStorageTags, additionalDiskSize, reservationDao, resourceLimitService) : null; - ) { + try { + checkedReservations = reserveStorageResourcesForVm(owner, diskOfferingId, diskSize, dataDiskInfoList, rootDiskOfferingId, offering, volumesSize); // verify security group ids if (securityGroupIdList != null) { @@ -4557,15 +4584,24 @@ private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, Stri UserVmVO vm = commitUserVm(zone, template, hostName, displayName, owner, diskOfferingId, diskSize, userData, userDataId, userDataDetails, caller, isDisplayVm, keyboard, accountId, userId, offering, isIso, sshPublicKeys, networkNicMap, id, instanceName, uuidName, hypervisorType, customParameters, dhcpOptionMap, - datadiskTemplateToDiskOfferringMap, userVmOVFPropertiesMap, dynamicScalingEnabled, vmType, rootDiskOfferingId, keypairnames, volume, snapshot); + datadiskTemplateToDiskOfferringMap, userVmOVFPropertiesMap, dynamicScalingEnabled, vmType, rootDiskOfferingId, keypairnames, dataDiskInfoList, volume, snapshot); assignInstanceToGroup(group, id); return vm; - } catch (ResourceAllocationException | CloudRuntimeException e) { + } catch (ResourceAllocationException | CloudRuntimeException e) { throw e; } catch (Exception e) { logger.error("error during resource reservation and allocation", e); throw new CloudRuntimeException(e); + } finally { + for (CheckedReservation checkedReservation : checkedReservations) { + try { + checkedReservation.close(); + } catch (Exception e) { + logger.error("error during resource reservation and allocation", e); + throw new CloudRuntimeException(e); + } + } } } @@ -4583,7 +4619,7 @@ private void assignInstanceToGroup(String group, long id) { } } - private long verifyAndGetDiskSize(DiskOfferingVO diskOffering, Long diskSize) { + private long verifyAndGetDiskSize(DiskOffering diskOffering, Long diskSize) { long size = 0l; if (diskOffering == null) { throw new InvalidParameterValueException("Specified disk offering cannot be found"); @@ -4698,7 +4734,8 @@ private UserVmVO commitUserVm(final boolean isImport, final DataCenter zone, fin final long accountId, final long userId, final ServiceOffering offering, final boolean isIso, final String sshPublicKeys, final LinkedHashMap> networkNicMap, final long id, final String instanceName, final String uuidName, final HypervisorType hypervisorType, final Map customParameters, final Map> extraDhcpOptionMap, final Map dataDiskTemplateToDiskOfferingMap, - final Map userVmOVFPropertiesMap, final VirtualMachine.PowerState powerState, final boolean dynamicScalingEnabled, String vmType, final Long rootDiskOfferingId, String sshkeypairs, Volume volume, Snapshot snapshot) throws InsufficientCapacityException { + final Map userVmOVFPropertiesMap, final VirtualMachine.PowerState powerState, final boolean dynamicScalingEnabled, String vmType, final Long rootDiskOfferingId, String sshkeypairs, + List dataDiskInfoList, Volume volume, Snapshot snapshot) throws InsufficientCapacityException { UserVmVO vm = new UserVmVO(id, instanceName, displayName, template.getId(), hypervisorType, template.getGuestOSId(), offering.isOfferHA(), offering.getLimitCpuUse(), owner.getDomainId(), owner.getId(), userId, offering.getId(), userData, userDataId, userDataDetails, hostName); vm.setUuid(uuidName); @@ -4816,7 +4853,7 @@ private UserVmVO commitUserVm(final boolean isImport, final DataCenter zone, fin orchestrateVirtualMachineCreate(vm, guestOSCategory, computeTags, rootDiskTags, plan, rootDiskSize, template, hostName, displayName, owner, diskOfferingId, diskSize, offering, isIso,networkNicMap, hypervisorType, extraDhcpOptionMap, dataDiskTemplateToDiskOfferingMap, - rootDiskOfferingId, volume, snapshot); + rootDiskOfferingId, dataDiskInfoList, volume, snapshot); } CallContext.current().setEventDetails("Vm Id: " + vm.getUuid()); @@ -4849,16 +4886,16 @@ private void orchestrateVirtualMachineCreate(UserVmVO vm, GuestOSCategoryVO gues ServiceOffering offering, boolean isIso, LinkedHashMap> networkNicMap, HypervisorType hypervisorType, Map> extraDhcpOptionMap, Map dataDiskTemplateToDiskOfferingMap, - Long rootDiskOfferingId, Volume volume, Snapshot snapshot) throws InsufficientCapacityException{ + Long rootDiskOfferingId, List dataDiskInfoList, Volume volume, Snapshot snapshot) throws InsufficientCapacityException{ try { if (isIso) { _orchSrvc.createVirtualMachineFromScratch(vm.getUuid(), Long.toString(owner.getAccountId()), vm.getIsoId().toString(), hostName, displayName, hypervisorType.name(), guestOSCategory.getName(), offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, - networkNicMap, plan, extraDhcpOptionMap, rootDiskOfferingId, volume, snapshot); + networkNicMap, plan, extraDhcpOptionMap, rootDiskOfferingId, dataDiskInfoList, volume, snapshot); } else { _orchSrvc.createVirtualMachine(vm.getUuid(), Long.toString(owner.getAccountId()), Long.toString(template.getId()), hostName, displayName, hypervisorType.name(), offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, networkNicMap, plan, rootDiskSize, extraDhcpOptionMap, - dataDiskTemplateToDiskOfferingMap, diskOfferingId, rootDiskOfferingId, volume, snapshot); + dataDiskTemplateToDiskOfferingMap, diskOfferingId, rootDiskOfferingId, dataDiskInfoList, volume, snapshot); } if (logger.isDebugEnabled()) { @@ -4979,14 +5016,15 @@ private UserVmVO commitUserVm(final DataCenter zone, final VirtualMachineTemplat final Long diskOfferingId, final Long diskSize, final String userData, Long userDataId, String userDataDetails, final Account caller, final Boolean isDisplayVm, final String keyboard, final long accountId, final long userId, final ServiceOfferingVO offering, final boolean isIso, final String sshPublicKeys, final LinkedHashMap> networkNicMap, final long id, final String instanceName, final String uuidName, final HypervisorType hypervisorType, final Map customParameters, final Map> extraDhcpOptionMap, final Map dataDiskTemplateToDiskOfferingMap, - Map userVmOVFPropertiesMap, final boolean dynamicScalingEnabled, String vmType, final Long rootDiskOfferingId, String sshkeypairs, Volume volume, Snapshot snapshot) throws InsufficientCapacityException { + Map> extraDhcpOptionMap, final Map dataDiskTemplateToDiskOfferingMap, + Map userVmOVFPropertiesMap, final boolean dynamicScalingEnabled, String vmType, final Long rootDiskOfferingId, String sshkeypairs, + List dataDiskInfoList, Volume volume, Snapshot snapshot) throws InsufficientCapacityException { return commitUserVm(false, zone, null, null, template, hostName, displayName, owner, diskOfferingId, diskSize, userData, userDataId, userDataDetails, isDisplayVm, keyboard, accountId, userId, offering, isIso, sshPublicKeys, networkNicMap, id, instanceName, uuidName, hypervisorType, customParameters, extraDhcpOptionMap, dataDiskTemplateToDiskOfferingMap, - userVmOVFPropertiesMap, null, dynamicScalingEnabled, vmType, rootDiskOfferingId, sshkeypairs, volume, snapshot); + userVmOVFPropertiesMap, null, dynamicScalingEnabled, vmType, rootDiskOfferingId, sshkeypairs, dataDiskInfoList, volume, snapshot); } public void validateRootDiskResize(final HypervisorType hypervisorType, Long rootDiskSize, VMTemplateVO templateVO, UserVmVO vm, final Map customParameters) throws InvalidParameterValueException @@ -6169,6 +6207,50 @@ public String finalizeUserData(String userData, Long userDataId, VirtualMachineT return null; } + private void verifyServiceOffering(BaseDeployVMCmd cmd, ServiceOffering serviceOffering) { + if (ServiceOffering.State.Inactive.equals(serviceOffering.getState())) { + throw new InvalidParameterValueException(String.format("Service offering is inactive: [%s].", serviceOffering.getUuid())); + } + + Long overrideDiskOfferingId = cmd.getOverrideDiskOfferingId(); + if (serviceOffering.getDiskOfferingStrictness() && overrideDiskOfferingId != null) { + throw new InvalidParameterValueException(String.format("Cannot override disk offering id %d since provided service offering is strictly mapped to its disk offering", overrideDiskOfferingId)); + } + + if (!serviceOffering.isDynamic()) { + for(String detail: cmd.getDetails().keySet()) { + if(detail.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || detail.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) || detail.equalsIgnoreCase(VmDetailConstants.MEMORY)) { + throw new InvalidParameterValueException("cpuNumber or cpuSpeed or memory should not be specified for static service offering"); + } + } + } + } + + private void verifyTemplate(BaseDeployVMCmd cmd, VirtualMachineTemplate template, Long serviceOfferingId) { + if (TemplateType.VNF.equals(template.getTemplateType())) { + vnfTemplateManager.validateVnfApplianceNics(template, cmd.getNetworkIds()); + } else if (cmd instanceof DeployVnfApplianceCmd) { + throw new InvalidParameterValueException("Can't deploy VNF appliance from a non-VNF template"); + } + + ServiceOfferingJoinVO svcOffering = serviceOfferingJoinDao.findById(serviceOfferingId); + + if (template.isDeployAsIs()) { + if (svcOffering != null && svcOffering.getRootDiskSize() != null && svcOffering.getRootDiskSize() > 0) { + throw new InvalidParameterValueException("Failed to deploy Virtual Machine as a service offering with root disk size specified cannot be used with a deploy as-is template"); + } + + if (cmd.getDetails().get("rootdisksize") != null) { + throw new InvalidParameterValueException("Overriding root disk size isn't supported for VMs deployed from deploy as-is templates"); + } + + // Bootmode and boottype are not supported on VMWare dpeloy-as-is templates (since 4.15) + if ((cmd.getBootMode() != null || cmd.getBootType() != null)) { + throw new InvalidParameterValueException("Boot type and boot mode are not supported on VMware for templates registered as deploy-as-is, as we honour what is defined in the template."); + } + } + } + @Override public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, StorageUnavailableException, ResourceAllocationException { @@ -6185,28 +6267,16 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE } Long serviceOfferingId = cmd.getServiceOfferingId(); + if (serviceOfferingId == null) { + throw new InvalidParameterValueException("Unable to execute API command deployvirtualmachine due to missing parameter serviceofferingid"); + } Long overrideDiskOfferingId = cmd.getOverrideDiskOfferingId(); ServiceOffering serviceOffering = _entityMgr.findById(ServiceOffering.class, serviceOfferingId); if (serviceOffering == null) { - throw new InvalidParameterValueException("Unable to find service offering: " + serviceOfferingId); - } - - if (ServiceOffering.State.Inactive.equals(serviceOffering.getState())) { - throw new InvalidParameterValueException(String.format("Service offering is inactive: [%s].", serviceOffering.getUuid())); - } - - if (serviceOffering.getDiskOfferingStrictness() && overrideDiskOfferingId != null) { - throw new InvalidParameterValueException(String.format("Cannot override disk offering id %d since provided service offering is strictly mapped to its disk offering", overrideDiskOfferingId)); - } - - if (!serviceOffering.isDynamic()) { - for(String detail: cmd.getDetails().keySet()) { - if(detail.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || detail.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) || detail.equalsIgnoreCase(VmDetailConstants.MEMORY)) { - throw new InvalidParameterValueException("cpuNumber or cpuSpeed or memory should not be specified for static service offering"); - } - } + throw new InvalidParameterValueException("Unable to find service offering: " + serviceOffering.getId()); } + verifyServiceOffering(cmd, serviceOffering); Account caller = CallContext.current().getCallingAccount(); Long callerId = caller.getId(); @@ -6234,44 +6304,21 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE overrideDiskOfferingId = volumeOfSnapshot.getDiskOfferingId(); } - boolean dynamicScalingEnabled = cmd.isDynamicScalingEnabled(); - VirtualMachineTemplate template = null; if (volume != null || snapshot != null) { template = _entityMgr.findByIdIncludingRemoved(VirtualMachineTemplate.class, templateId); } else { template = _entityMgr.findById(VirtualMachineTemplate.class, templateId); } - // Make sure a valid template ID was specified - if (template == null) { - throw new InvalidParameterValueException("Unable to use template " + templateId); - } - if (TemplateType.VNF.equals(template.getTemplateType())) { - vnfTemplateManager.validateVnfApplianceNics(template, cmd.getNetworkIds()); - } else if (cmd instanceof DeployVnfApplianceCmd) { - throw new InvalidParameterValueException("Can't deploy VNF appliance from a non-VNF template"); - } - if (cmd.isVolumeOrSnapshotProvided() && (!(HypervisorType.KVM.equals(template.getHypervisorType()) || HypervisorType.KVM.equals(cmd.getHypervisor())))) { throw new InvalidParameterValueException("Deploying a virtual machine with existing volume/snapshot is supported only from KVM hypervisors"); } - ServiceOfferingJoinVO svcOffering = serviceOfferingJoinDao.findById(serviceOfferingId); - - if (template.isDeployAsIs()) { - if (svcOffering != null && svcOffering.getRootDiskSize() != null && svcOffering.getRootDiskSize() > 0) { - throw new InvalidParameterValueException("Failed to deploy Virtual Machine as a service offering with root disk size specified cannot be used with a deploy as-is template"); - } - - if (cmd.getDetails().get("rootdisksize") != null) { - throw new InvalidParameterValueException("Overriding root disk size isn't supported for VMs deployed from deploy as-is templates"); - } - - // Bootmode and boottype are not supported on VMWare dpeloy-as-is templates (since 4.15) - if ((cmd.getBootMode() != null || cmd.getBootType() != null)) { - throw new InvalidParameterValueException("Boot type and boot mode are not supported on VMware for templates registered as deploy-as-is, as we honour what is defined in the template."); - } + // Make sure a valid template ID was specified + if (template == null) { + throw new InvalidParameterValueException("Unable to use template " + templateId); } + verifyTemplate(cmd, template, serviceOfferingId); Long diskOfferingId = cmd.getDiskOfferingId(); DiskOffering diskOffering = null; @@ -6285,6 +6332,11 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE } } + List dataDiskInfoList = cmd.getDataDiskInfoList(); + if (dataDiskInfoList != null && diskOfferingId != null) { + new InvalidParameterValueException("Cannot specify both disk offering id and data disk offering details"); + } + if (!zone.isLocalStorageEnabled()) { DiskOffering diskOfferingMappedInServiceOffering = _entityMgr.findById(DiskOffering.class, serviceOffering.getDiskOfferingId()); if (diskOfferingMappedInServiceOffering.isUseLocalStorage()) { @@ -6295,25 +6347,48 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE } } - boolean isLeaseFeatureEnabled = VMLeaseManager.InstanceLeaseEnabled.value(); - if (isLeaseFeatureEnabled) { - validateLeaseProperties(cmd.getLeaseDuration(), cmd.getLeaseExpiryAction()); - } - List networkIds = cmd.getNetworkIds(); LinkedHashMap userVmNetworkMap = getVmOvfNetworkMapping(zone, owner, template, cmd.getVmNetworkMap()); if (MapUtils.isNotEmpty(userVmNetworkMap)) { networkIds = new ArrayList<>(userVmNetworkMap.values()); } - String userData = cmd.getUserData(); - Long userDataId = cmd.getUserdataId(); + return createVirtualMachine(cmd, zone, owner, serviceOffering, template, cmd.getHypervisor(), diskOfferingId, cmd.getSize(), overrideDiskOfferingId, dataDiskInfoList, networkIds, cmd.getIpToNetworkMap(), volume, snapshot); + } + + private UserVm createVirtualMachine(BaseDeployVMCmd cmd, DataCenter zone, Account owner, ServiceOffering serviceOffering, VirtualMachineTemplate template, + HypervisorType hypervisor, Long diskOfferingId, Long size, Long overrideDiskOfferingId, List dataDiskInfoList, + List networkIds, Map ipToNetworkMap, Volume volume, Snapshot snapshot) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, ResourceAllocationException { + + ServiceOfferingJoinVO svcOffering = serviceOfferingJoinDao.findById(serviceOffering.getId()); + boolean isLeaseFeatureEnabled = VMLeaseManager.InstanceLeaseEnabled.value(); + if (isLeaseFeatureEnabled) { + validateLeaseProperties(cmd.getLeaseDuration(), cmd.getLeaseExpiryAction()); + } + + String userData = null; + Long userDataId = null; String userDataDetails = null; - if (MapUtils.isNotEmpty(cmd.getUserdataDetails())) { - userDataDetails = cmd.getUserdataDetails().toString(); + List sshKeyPairNames = new ArrayList(); + if (cmd instanceof CreateVMFromBackupCmd) { + if (cmd.getUserData() != null) { + throw new InvalidParameterValueException("User data not supported for instance created from backup"); + } + } else { + userData = cmd.getUserData(); + userDataId = cmd.getUserdataId(); + userDataDetails = null; + if (MapUtils.isNotEmpty(cmd.getUserdataDetails())) { + userDataDetails = cmd.getUserdataDetails().toString(); + } + userData = finalizeUserData(userData, userDataId, template); + userData = userDataManager.validateUserData(userData, cmd.getHttpMethod()); + + sshKeyPairNames = cmd.getSSHKeyPairNames(); } - userData = finalizeUserData(userData, userDataId, template); - userData = userDataManager.validateUserData(userData, cmd.getHttpMethod()); + + Account caller = CallContext.current().getCallingAccount(); + Long callerId = caller.getId(); boolean isRootAdmin = _accountService.isRootAdmin(callerId); @@ -6327,9 +6402,8 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE String displayName = cmd.getDisplayName(); UserVm vm = null; IpAddresses addrs = new IpAddresses(ipAddress, ip6Address, macAddress); - Long size = cmd.getSize(); + boolean dynamicScalingEnabled = cmd.isDynamicScalingEnabled(); String group = cmd.getGroup(); - List sshKeyPairNames = cmd.getSSHKeyPairNames(); Boolean displayVm = cmd.isDisplayVm(); String keyboard = cmd.getKeyboard(); Map dataDiskTemplateToDiskOfferingMap = cmd.getDataDiskTemplateToDiskOfferingMap(); @@ -6339,7 +6413,7 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE throw new InvalidParameterValueException("Can't specify network Ids in Basic zone"); } else { vm = createBasicSecurityGroupVirtualMachine(zone, serviceOffering, template, getSecurityGroupIdList(cmd, zone, template, owner), owner, name, displayName, diskOfferingId, - size , group , cmd.getHypervisor(), cmd.getHttpMethod(), userData, userDataId, userDataDetails, sshKeyPairNames, cmd.getIpToNetworkMap(), addrs, displayVm , keyboard , cmd.getAffinityGroupIdList(), + size , dataDiskInfoList, group , hypervisor, cmd.getHttpMethod(), userData, userDataId, userDataDetails, sshKeyPairNames, ipToNetworkMap, addrs, displayVm , keyboard , cmd.getAffinityGroupIdList(), cmd.getDetails(), cmd.getCustomId(), cmd.getDhcpOptionsMap(), dataDiskTemplateToDiskOfferingMap, userVmOVFProperties, dynamicScalingEnabled, overrideDiskOfferingId, volume, snapshot); } @@ -6347,7 +6421,7 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE if (_networkModel.checkSecurityGroupSupportForNetwork(owner, zone, networkIds, cmd.getSecurityGroupIdList())) { vm = createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, template, networkIds, getSecurityGroupIdList(cmd, zone, template, owner), owner, name, - displayName, diskOfferingId, size, group, cmd.getHypervisor(), cmd.getHttpMethod(), userData, userDataId, userDataDetails, sshKeyPairNames, cmd.getIpToNetworkMap(), addrs, displayVm, keyboard, + displayName, diskOfferingId, size, dataDiskInfoList, group, hypervisor, cmd.getHttpMethod(), userData, userDataId, userDataDetails, sshKeyPairNames, ipToNetworkMap, addrs, displayVm, keyboard, cmd.getAffinityGroupIdList(), cmd.getDetails(), cmd.getCustomId(), cmd.getDhcpOptionsMap(), dataDiskTemplateToDiskOfferingMap, userVmOVFProperties, dynamicScalingEnabled, overrideDiskOfferingId, null, volume, snapshot); @@ -6355,8 +6429,8 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE if (cmd.getSecurityGroupIdList() != null && !cmd.getSecurityGroupIdList().isEmpty()) { throw new InvalidParameterValueException("Can't create vm with security groups; security group feature is not enabled per zone"); } - vm = createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, name, displayName, diskOfferingId, size, group, - cmd.getHypervisor(), cmd.getHttpMethod(), userData, userDataId, userDataDetails, sshKeyPairNames, cmd.getIpToNetworkMap(), addrs, displayVm, keyboard, cmd.getAffinityGroupIdList(), cmd.getDetails(), + vm = createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, name, displayName, diskOfferingId, size, dataDiskInfoList, group, + hypervisor, cmd.getHttpMethod(), userData, userDataId, userDataDetails, sshKeyPairNames, ipToNetworkMap, addrs, displayVm, keyboard, cmd.getAffinityGroupIdList(), cmd.getDetails(), cmd.getCustomId(), cmd.getDhcpOptionsMap(), dataDiskTemplateToDiskOfferingMap, userVmOVFProperties, dynamicScalingEnabled, null, overrideDiskOfferingId, volume, snapshot); if (cmd instanceof DeployVnfApplianceCmd) { vnfTemplateManager.createIsolatedNetworkRulesForVnfAppliance(zone, template, owner, vm, (DeployVnfApplianceCmd) cmd); @@ -6365,7 +6439,7 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE } // check if this templateId has a child ISO - List child_templates = _templateDao.listByParentTemplatetId(templateId); + List child_templates = _templateDao.listByParentTemplatetId(template.getId()); for (VMTemplateVO tmpl: child_templates){ if (tmpl.getFormat() == Storage.ImageFormat.ISO){ logger.info("MDOV trying to attach disk {} to the VM {}", tmpl, vm); @@ -6385,10 +6459,10 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE } if (cmd.getCopyImageTags()) { - VMTemplateVO templateOrIso = _templateDao.findById(templateId); + VMTemplateVO templateOrIso = _templateDao.findById(template.getId()); if (templateOrIso != null) { final ResourceTag.ResourceObjectType templateType = (templateOrIso.getFormat() == ImageFormat.ISO) ? ResourceTag.ResourceObjectType.ISO : ResourceTag.ResourceObjectType.Template; - final List resourceTags = resourceTagDao.listBy(templateId, templateType); + final List resourceTags = resourceTagDao.listBy(template.getId(), templateType); for (ResourceTag resourceTag : resourceTags) { final ResourceTagVO copyTag = new ResourceTagVO(resourceTag.getKey(), resourceTag.getValue(), resourceTag.getAccountId(), resourceTag.getDomainId(), vm.getId(), ResourceTag.ResourceObjectType.UserVm, resourceTag.getCustomer(), vm.getUuid()); resourceTagDao.persist(copyTag); @@ -9331,7 +9405,7 @@ public UserVm importVM(final DataCenter zone, final Host host, final VirtualMach null, null, userData, null, null, isDisplayVm, keyboard, accountId, userId, serviceOffering, template.getFormat().equals(ImageFormat.ISO), sshPublicKeys, networkNicMap, id, instanceName, uuidName, hypervisorType, customParameters, - null, null, null, powerState, dynamicScalingEnabled, null, serviceOffering.getDiskOfferingId(), null, null, null); + null, null, null, powerState, dynamicScalingEnabled, null, serviceOffering.getDiskOfferingId(), null, null, null, null); } @Override @@ -9376,6 +9450,234 @@ public boolean unmanageUserVM(Long vmId) { return true; } + private void updateDetailsWithRootDiskAttributes(Map details, VmDiskInfo rootVmDiskInfo) { + details.put(VmDetailConstants.ROOT_DISK_SIZE, rootVmDiskInfo.getSize().toString()); + if (rootVmDiskInfo.getMinIops() != null) { + details.put(MIN_IOPS, rootVmDiskInfo.getMinIops().toString()); + } + if (rootVmDiskInfo.getMaxIops() != null) { + details.put(MAX_IOPS, rootVmDiskInfo.getMaxIops().toString()); + } + } + + private void checkRootDiskSizeAgainstBackup(Long instanceVolumeSize,DiskOffering rootDiskOffering, Long backupVolumeSize) { + Long instanceRootDiskSize = rootDiskOffering.isCustomized() ? instanceVolumeSize : rootDiskOffering.getDiskSize() / GiB_TO_BYTES; + if (instanceRootDiskSize < backupVolumeSize) { + throw new InvalidParameterValueException( + String.format("Instance volume root disk size %d[GiB] cannot be less than the backed-up volume size %d[GiB].", + instanceVolumeSize, backupVolumeSize)); + } + } + + @Override + public UserVm allocateVMFromBackup(CreateVMFromBackupCmd cmd) throws InsufficientCapacityException, ResourceAllocationException, ResourceUnavailableException { + if (!backupManager.canCreateInstanceFromBackup(cmd.getBackupId())) { + throw new CloudRuntimeException("Create instance from backup is not supported for this provider."); + } + DataCenter zone = _dcDao.findById(cmd.getZoneId()); + if (zone == null) { + throw new InvalidParameterValueException("Unable to find zone by id=" + cmd.getZoneId()); + } + + BackupVO backup = backupDao.findById(cmd.getBackupId()); + if (backup == null) { + throw new InvalidParameterValueException("Backup " + cmd.getBackupId() + " does not exist"); + } + if (backup.getZoneId() != cmd.getZoneId()) { + throw new InvalidParameterValueException("Instance should be created in the same zone as the backup"); + } + backupManager.validateBackupForZone(backup.getZoneId()); + backupDao.loadDetails(backup); + + verifyDetails(cmd.getDetails()); + + UserVmVO backupVm = _vmDao.findByIdIncludingRemoved(backup.getVmId()); + HypervisorType hypervisorType = backupVm.getHypervisorType(); + + Long serviceOfferingId = cmd.getServiceOfferingId(); + ServiceOffering serviceOffering; + if (serviceOfferingId != null) { + serviceOffering = serviceOfferingDao.findById(serviceOfferingId); + if (serviceOffering == null) { + throw new InvalidParameterValueException("Unable to find service offering: " + serviceOffering.getId()); + } + } else { + String serviceOfferingUuid = backup.getDetail(ApiConstants.SERVICE_OFFERING_ID); + if (serviceOfferingUuid == null) { + throw new CloudRuntimeException("Backup doesn't contain service offering uuid. Please specify a valid service offering id while creating the instance"); + } + serviceOffering = serviceOfferingDao.findByUuid(serviceOfferingUuid); + if (serviceOffering == null) { + throw new CloudRuntimeException("Unable to find service offering with the uuid stored in backup. Please specify a valid service offering id while creating instance"); + } + } + verifyServiceOffering(cmd, serviceOffering); + + VirtualMachineTemplate template; + if (cmd.getTemplateId() != null) { + Long templateId = cmd.getTemplateId(); + template = _templateDao.findById(templateId); + if (template == null) { + throw new InvalidParameterValueException("Unable to use template " + templateId); + } + } else { + String templateUuid = backup.getDetail(ApiConstants.TEMPLATE_ID); + if (templateUuid == null) { + throw new CloudRuntimeException("Backup doesn't contain Template uuid. Please specify a valid Template/ISO while creating the instance"); + } + template = _templateDao.findByUuid(templateUuid); + if (template == null) { + throw new CloudRuntimeException("Unable to find template associated with the backup. Please specify a valid Template/ISO while creating instance"); + } + } + verifyTemplate(cmd, template, serviceOffering.getId()); + + Long size = cmd.getSize(); + + Long diskOfferingId = cmd.getDiskOfferingId(); + Boolean isIso = template.getFormat().equals(ImageFormat.ISO); + if (diskOfferingId != null) { + if (!isIso) { + throw new InvalidParameterValueException(ApiConstants.DISK_OFFERING_ID + " parameter is supported for creating instance from backup only for ISO. For creating VMs with templates, please use the parameter " + ApiConstants.DATADISKS_DETAILS); + } + DiskOffering diskOffering = _diskOfferingDao.findById(diskOfferingId); + if (diskOffering == null) { + throw new InvalidParameterValueException("Unable to find disk offering " + diskOfferingId); + } + if (diskOffering.isComputeOnly()) { + throw new InvalidParameterValueException(String.format("The disk offering %s provided is directly mapped to a service offering, please provide an individual disk offering", diskOffering)); + } + } + + Long overrideDiskOfferingId = cmd.getOverrideDiskOfferingId(); + + VmDiskInfo rootVmDiskInfoFromBackup = backupManager.getRootDiskInfoFromBackup(backup); + + if (isIso) { + if (diskOfferingId == null) { + diskOfferingId = rootVmDiskInfoFromBackup.getDiskOffering().getId(); + updateDetailsWithRootDiskAttributes(cmd.getDetails(), rootVmDiskInfoFromBackup); + size = rootVmDiskInfoFromBackup.getSize(); + } else { + DiskOffering rootDiskOffering = _diskOfferingDao.findById(diskOfferingId); + checkRootDiskSizeAgainstBackup(size, rootDiskOffering, rootVmDiskInfoFromBackup.getSize()); + } + } else { + if (overrideDiskOfferingId == null) { + overrideDiskOfferingId = serviceOffering.getDiskOfferingId(); + updateDetailsWithRootDiskAttributes(cmd.getDetails(), rootVmDiskInfoFromBackup); + } else { + DiskOffering overrideDiskOffering = _diskOfferingDao.findById(overrideDiskOfferingId); + if (overrideDiskOffering.isComputeOnly()) { + updateDetailsWithRootDiskAttributes(cmd.getDetails(), rootVmDiskInfoFromBackup); + } else { + String diskSizeFromDetails = cmd.getDetails().get(VmDetailConstants.ROOT_DISK_SIZE); + Long rootDiskSize = diskSizeFromDetails == null ? null : Long.parseLong(diskSizeFromDetails); + checkRootDiskSizeAgainstBackup(rootDiskSize, overrideDiskOffering, rootVmDiskInfoFromBackup.getSize()); + } + } + } + + List dataDiskInfoList = cmd.getDataDiskInfoList(); + if (dataDiskInfoList != null) { + backupManager.checkVmDisksSizeAgainstBackup(dataDiskInfoList, backup); + } else { + dataDiskInfoList = backupManager.getDataDiskInfoListFromBackup(backup); + } + + List networkIds = cmd.getNetworkIds(); + Account owner = _accountService.getActiveAccountById(cmd.getEntityOwnerId()); + LinkedHashMap userVmNetworkMap = getVmOvfNetworkMapping(zone, owner, template, cmd.getVmNetworkMap()); + if (MapUtils.isNotEmpty(userVmNetworkMap)) { + networkIds = new ArrayList<>(userVmNetworkMap.values()); + } + + Map ipToNetworkMap = cmd.getIpToNetworkMap(); + if (networkIds == null && ipToNetworkMap == null) { + networkIds = new ArrayList(); + ipToNetworkMap = backupManager.getIpToNetworkMapFromBackup(backup, cmd.getPreserveIp(), networkIds); + } + + UserVm vm = createVirtualMachine(cmd, zone, owner, serviceOffering, template, hypervisorType, diskOfferingId, size, overrideDiskOfferingId, dataDiskInfoList, networkIds, ipToNetworkMap, null, null); + + String vmSettingsFromBackup = backup.getDetail(ApiConstants.VM_SETTINGS); + if (vm != null && vmSettingsFromBackup != null) { + UserVmVO vmVO = _vmDao.findById(vm.getId()); + Map details = vmInstanceDetailsDao.listDetailsKeyPairs(vm.getId()); + vmVO.setDetails(details); + + Type type = new TypeToken>(){}.getType(); + Map vmDetailsFromBackup = new Gson().fromJson(vmSettingsFromBackup, type); + for (Entry entry : vmDetailsFromBackup.entrySet()) { + if (!details.containsKey(entry.getKey())) { + vmVO.setDetail(entry.getKey(), entry.getValue()); + } + } + _vmDao.saveDetails(vmVO); + } + + return vm; + } + + @Override + public UserVm restoreVMFromBackup(CreateVMFromBackupCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException, ResourceAllocationException { + long vmId = cmd.getEntityId(); + Map diskOfferingMap = cmd.getDataDiskTemplateToDiskOfferingMap(); + Map additonalParams = new HashMap<>(); + UserVm vm; + + try { + vm = startVirtualMachine(vmId, null, null, null, diskOfferingMap, additonalParams, null); + + boolean status = stopVirtualMachine(CallContext.current().getCallingUserId(), vm.getId()) ; + if (!status) { + UserVmVO vmVO = _vmDao.findById(vmId); + expunge(vmVO); + logger.debug("Successfully cleaned up Instance {} after create Instance from backup failed", vmId); + throw new CloudRuntimeException("Unable to stop the Instance before restore"); + } + + Long isoId = vm.getIsoId(); + if (isoId != null) { + UserVmVO vmVO = _vmDao.findById(vmId); + vmVO.setIsoId(null); + _vmDao.update(vm.getId(), vmVO); + } + + backupManager.restoreBackupToVM(cmd.getBackupId(), vmId); + + } catch (CloudRuntimeException e) { + UserVmVO vmVO = _vmDao.findById(vmId); + try { + expunge(vmVO); + logger.debug("Successfully cleaned up Instance {} after create Instance from backup failed", vmId); + } catch (Exception cleanupException) { + logger.debug("Failed to cleanup Instance {} after create Instance from backup failed", vmId, cleanupException); + } + throw e; + } + + Account owner = _accountService.getActiveAccountById(cmd.getEntityOwnerId()); + UserVmVO userVm = _vmDao.findById(vmId); + + List sshKeyPairNames = cmd.getSSHKeyPairNames(); + if (sshKeyPairNames != null && !sshKeyPairNames.isEmpty()) { + vm = resetVMSSHKeyInternal(userVm, owner, sshKeyPairNames); + } + + if (cmd.getStartVm()) { + Long podId = null; + Long clusterId = null; + if (cmd instanceof CreateVMFromBackupCmdByAdmin) { + CreateVMFromBackupCmdByAdmin adminCmd = (CreateVMFromBackupCmdByAdmin)cmd; + podId = adminCmd.getPodId(); + clusterId = adminCmd.getClusterId(); + } + vm = startVirtualMachine(vmId, podId, clusterId, cmd.getHostId(), diskOfferingMap, additonalParams, cmd.getDeploymentPlanner()); + } + return vm; + } + /* Generate usage events related to unmanaging a VM */ diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 52a12a6ff9b0..889db69d353a 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -16,25 +16,28 @@ // under the License. package org.apache.cloudstack.backup; +import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.TimeZone; import java.util.Timer; import java.util.TimerTask; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.serializer.GsonHelper; -import com.google.gson.reflect.TypeToken; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.InternalIdentity; @@ -43,6 +46,7 @@ import org.apache.cloudstack.api.command.admin.backup.ListBackupProviderOfferingsCmd; import org.apache.cloudstack.api.command.admin.backup.ListBackupProvidersCmd; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; +import org.apache.cloudstack.api.command.admin.vm.CreateVMFromBackupCmdByAdmin; import org.apache.cloudstack.api.command.user.backup.AssignVirtualMachineToBackupOfferingCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; @@ -58,7 +62,10 @@ import org.apache.cloudstack.api.command.user.backup.repository.AddBackupRepositoryCmd; import org.apache.cloudstack.api.command.user.backup.repository.DeleteBackupRepositoryCmd; import org.apache.cloudstack.api.command.user.backup.repository.ListBackupRepositoriesCmd; +import org.apache.cloudstack.api.command.user.vm.CreateVMFromBackupCmd; +import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupDetailsDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.backup.dao.BackupScheduleDao; import org.apache.cloudstack.context.CallContext; @@ -82,9 +89,16 @@ import com.cloud.alert.AlertManager; import com.cloud.api.ApiDispatcher; import com.cloud.api.ApiGsonHelper; +import com.cloud.api.query.dao.UserVmJoinDao; +import com.cloud.api.query.vo.UserVmJoinVO; +import com.cloud.capacity.Capacity; +import com.cloud.capacity.CapacityVO; import com.cloud.configuration.Resource; import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; import com.cloud.event.ActionEvent; import com.cloud.event.ActionEventUtils; import com.cloud.event.EventTypes; @@ -93,24 +107,38 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.HypervisorGuru; import com.cloud.hypervisor.HypervisorGuruManager; +import com.cloud.network.Network; +import com.cloud.network.NetworkService; +import com.cloud.network.dao.NetworkDao; +import com.cloud.offering.DiskOffering; +import com.cloud.offering.ServiceOffering; import com.cloud.projects.Project; +import com.cloud.serializer.GsonHelper; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountService; +import com.cloud.user.AccountVO; import com.cloud.user.DomainManager; import com.cloud.user.ResourceLimitService; import com.cloud.user.User; +import com.cloud.user.dao.AccountDao; import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; @@ -129,18 +157,24 @@ import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; +import com.cloud.vm.VMInstanceDetailVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.VmDiskInfo; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; public class BackupManagerImpl extends ManagerBase implements BackupManager { @Inject private BackupDao backupDao; @Inject + private BackupDetailsDao backupDetailsDao; + @Inject private BackupScheduleDao backupScheduleDao; @Inject private BackupOfferingDao backupOfferingDao; @@ -153,6 +187,10 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { @Inject private DomainManager domainManager; @Inject + private AccountDao accountDao; + @Inject + private DomainDao domainDao; + @Inject private VolumeDao volumeDao; @Inject private DataCenterDao dataCenterDao; @@ -169,6 +207,18 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { @Inject private UserVmDao userVmDao; @Inject + private ServiceOfferingDao serviceOfferingDao; + @Inject + private VMTemplateDao vmTemplateDao; + @Inject + private UserVmJoinDao userVmJoinDao; + @Inject + private VMInstanceDetailsDao vmInstanceDetailsDao; + @Inject + private NetworkDao networkDao; + @Inject + private NetworkService networkService; + @Inject private ApiDispatcher apiDispatcher; @Inject private AsyncJobManager asyncJobManager; @@ -201,7 +251,7 @@ public List listBackupProviderOfferings(final Long zoneId) { if (zoneId == null || zoneId < 1) { throw new CloudRuntimeException("Invalid zone ID passed"); } - validateForZone(zoneId); + validateBackupForZone(zoneId); final Account account = CallContext.current().getCallingAccount(); if (!accountService.isRootAdmin(account.getId())) { throw new PermissionDeniedException("Parameter external can only be specified by a Root Admin, permission denied"); @@ -214,7 +264,7 @@ public List listBackupProviderOfferings(final Long zoneId) { @Override @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_IMPORT_OFFERING, eventDescription = "importing backup offering", async = true) public BackupOffering importBackupOffering(final ImportBackupOfferingCmd cmd) { - validateForZone(cmd.getZoneId()); + validateBackupForZone(cmd.getZoneId()); final BackupOffering existingOffering = backupOfferingDao.findByExternalId(cmd.getExternalId(), cmd.getZoneId()); if (existingOffering != null) { throw new CloudRuntimeException("A backup offering with external ID " + cmd.getExternalId() + " already exists"); @@ -285,19 +335,81 @@ public boolean deleteBackupOffering(final Long offeringId) { throw new CloudRuntimeException("Could not find a backup offering with id: " + offeringId); } - if (vmInstanceDao.listByZoneWithBackups(offering.getZoneId(), offering.getId()).size() > 0) { + if (backupDao.listByOfferingId(offering.getId()).size() > 0) { + throw new CloudRuntimeException("Backup Offering cannot be removed as it has backups associated with it."); + } + + if (vmInstanceDao.listByZoneAndBackupOffering(offering.getZoneId(), offering.getId()).size() > 0) { throw new CloudRuntimeException("Backup offering is assigned to VMs, remove the assignment(s) in order to remove the offering."); } - validateForZone(offering.getZoneId()); + validateBackupForZone(offering.getZoneId()); return backupOfferingDao.remove(offering.getId()); } - public static String createVolumeInfoFromVolumes(List vmVolumes) { + private String getNicDetailsAsJson(final Long vmId) { + final List userVmJoinVOs = userVmJoinDao.searchByIds(vmId); + if (userVmJoinVOs != null && !userVmJoinVOs.isEmpty()) { + final List> nics = new ArrayList<>(); + final Set seen = new HashSet<>(); + for (UserVmJoinVO userVmJoinVO : userVmJoinVOs) { + Map nicInfo = new HashMap<>(); + String key = userVmJoinVO.getNetworkUuid(); + if (seen.add(key)) { + nicInfo.put(ApiConstants.NETWORK_ID, userVmJoinVO.getNetworkUuid()); + nicInfo.put(ApiConstants.IP_ADDRESS, userVmJoinVO.getIpAddress()); + nicInfo.put(ApiConstants.IP6_ADDRESS, userVmJoinVO.getIp6Address()); + nicInfo.put(ApiConstants.MAC_ADDRESS, userVmJoinVO.getMacAddress()); + nics.add(nicInfo); + } + } + if (!nics.isEmpty()) { + return new Gson().toJson(nics); + } + } + return null; + } + + @Override + public Map getBackupDetailsFromVM(VirtualMachine vm) { + HashMap details = new HashMap<>(); + + ServiceOffering serviceOffering = serviceOfferingDao.findById(vm.getServiceOfferingId()); + details.put(ApiConstants.SERVICE_OFFERING_ID, serviceOffering.getUuid()); + VirtualMachineTemplate template = vmTemplateDao.findById(vm.getTemplateId()); + details.put(ApiConstants.TEMPLATE_ID, template.getUuid()); + + List vmDetails = vmInstanceDetailsDao.listDetails(vm.getId()); + HashMap settings = new HashMap<>(); + for (VMInstanceDetailVO detail : vmDetails) { + settings.put(detail.getName(), detail.getValue()); + } + if (!settings.isEmpty()) { + details.put(ApiConstants.VM_SETTINGS, new Gson().toJson(settings)); + } + + String nicsJson = getNicDetailsAsJson(vm.getId()); + if (nicsJson != null) { + details.put(ApiConstants.NICS, nicsJson); + } + return details; + } + + @Override + public String getBackupNameFromVM(VirtualMachine vm) { + String displayTime = DateUtil.displayDateInTimezone(DateUtil.GMT_TIMEZONE, new Date()); + return (vm.getHostName() + '-' + displayTime); + } + + @Override + public String createVolumeInfoFromVolumes(List vmVolumes) { List list = new ArrayList<>(); - vmVolumes.sort(Comparator.comparing(VolumeVO::getDeviceId)); - for (VolumeVO vol : vmVolumes) { - list.add(new Backup.VolumeInfo(vol.getUuid(), vol.getPath(), vol.getVolumeType(), vol.getSize())); + vmVolumes.sort(Comparator.comparing(Volume::getDeviceId)); + for (Volume vol : vmVolumes) { + DiskOfferingVO diskOffering = diskOfferingDao.findById(vol.getDiskOfferingId()); + Backup.VolumeInfo volumeInfo = new Backup.VolumeInfo(vol.getUuid(), vol.getPath(), vol.getVolumeType(), vol.getSize(), + vol.getDeviceId(), diskOffering.getUuid(), vol.getMinIops(), vol.getMaxIops()); + list.add(volumeInfo); } return new Gson().toJson(list.toArray(), Backup.VolumeInfo[].class); } @@ -311,7 +423,7 @@ public boolean assignVMToBackupOffering(Long vmId, Long offeringId) { throw new CloudRuntimeException("VM is not in running or stopped state"); } - validateForZone(vm.getDataCenterId()); + validateBackupForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); @@ -339,7 +451,7 @@ public VMInstanceVO doInTransaction(final TransactionStatus status) { try { long vmId = vm.getId(); vm.setBackupOfferingId(offering.getId()); - vm.setBackupVolumes(createVolumeInfoFromVolumes(volumeDao.findByInstance(vmId))); + vm.setBackupVolumes(createVolumeInfoFromVolumes(new ArrayList<>(volumeDao.findByInstance(vmId)))); if (!backupProvider.assignVMToBackupOffering(vm, offering)) { throw new CloudRuntimeException("Failed to assign the VM to the backup offering, please try removing the assignment and try again."); @@ -377,7 +489,7 @@ public boolean removeVMFromBackupOffering(final Long vmId, final boolean forced) throw new CloudRuntimeException(String.format("Can't find any VM with ID: [%s].", vmId)); } - validateForZone(vm.getDataCenterId()); + validateBackupForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); final BackupOfferingVO offering = backupOfferingDao.findById(vm.getBackupOfferingId()); @@ -400,6 +512,7 @@ public boolean removeVMFromBackupOffering(final Long vmId, final boolean forced) boolean result = false; try { result = backupProvider.removeVMFromBackupOffering(vm); + Long backupOfferingId = vm.getBackupOfferingId(); vm.setBackupOfferingId(null); vm.setBackupVolumes(null); vm.setBackupExternalId(null); @@ -410,9 +523,12 @@ public boolean removeVMFromBackupOffering(final Long vmId, final boolean forced) } } if ((result || forced) && vmInstanceDao.update(vm.getId(), vm)) { - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), - "Backup-" + vm.getHostName() + "-" + vm.getUuid(), vm.getBackupOfferingId(), null, null, - Backup.class.getSimpleName(), vm.getUuid()); + final List backups = backupDao.listByVmId(null, vm.getId()); + if (backups.size() == 0) { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVED_AND_BACKUPS_DELETED, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), + "Backup-" + vm.getHostName() + "-" + vm.getUuid(), backupOfferingId, null, null, + Backup.class.getSimpleName(), vm.getUuid()); + } final List backupSchedules = backupScheduleDao.listByVM(vm.getId()); for(BackupSchedule backupSchedule: backupSchedules) { backupScheduleDao.remove(backupSchedule.getId()); @@ -438,7 +554,7 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { } final VMInstanceVO vm = findVmById(vmId); - validateForZone(vm.getDataCenterId()); + validateBackupForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); if (vm.getBackupOfferingId() == null) { @@ -452,6 +568,10 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { final int maxBackups = validateAndGetDefaultBackupRetentionIfRequired(cmd.getMaxBackups(), offering, vm); + if (!"nas".equals(offering.getProvider()) && cmd.getQuiesceVM() != null) { + throw new InvalidParameterValueException("Quiesce VM option is supported only for NAS backup provider"); + } + final String timezoneId = timeZone.getID(); if (!timezoneId.equals(cmd.getTimezone())) { logger.warn("Using timezone: " + timezoneId + " for running this snapshot policy as an equivalent of " + cmd.getTimezone()); @@ -466,7 +586,7 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { final BackupScheduleVO schedule = backupScheduleDao.findByVMAndIntervalType(vmId, intervalType); if (schedule == null) { - return backupScheduleDao.persist(new BackupScheduleVO(vmId, intervalType, scheduleString, timezoneId, nextDateTime, maxBackups)); + return backupScheduleDao.persist(new BackupScheduleVO(vmId, intervalType, scheduleString, timezoneId, nextDateTime, maxBackups, cmd.getQuiesceVM())); } schedule.setScheduleType((short) intervalType.ordinal()); @@ -474,6 +594,7 @@ public BackupSchedule configureBackupSchedule(CreateBackupScheduleCmd cmd) { schedule.setTimezone(timezoneId); schedule.setScheduledTimestamp(nextDateTime); schedule.setMaxBackups(maxBackups); + schedule.setQuiesceVM(cmd.getQuiesceVM()); backupScheduleDao.update(schedule.getId(), schedule); return backupScheduleDao.findById(schedule.getId()); } @@ -518,7 +639,7 @@ protected int validateAndGetDefaultBackupRetentionIfRequired(Integer maxBackups, @Override public List listBackupSchedule(final Long vmId) { final VMInstanceVO vm = findVmById(vmId); - validateForZone(vm.getDataCenterId()); + validateBackupForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); return backupScheduleDao.listByVM(vmId).stream().map(BackupSchedule.class::cast).collect(Collectors.toList()); @@ -556,7 +677,7 @@ public boolean deleteBackupSchedule(DeleteBackupScheduleCmd cmd) { */ protected void checkCallerAccessToBackupScheduleVm(long vmId) { VMInstanceVO vm = findVmById(vmId); - validateForZone(vm.getDataCenterId()); + validateBackupForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); } @@ -577,9 +698,11 @@ protected boolean deleteAllVmBackupSchedules(long vmId) { @Override @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_CREATE, eventDescription = "creating VM backup", async = true) - public boolean createBackup(final Long vmId, Object job) throws ResourceAllocationException { + public boolean createBackup(CreateBackupCmd cmd, Object job) throws ResourceAllocationException { + Long vmId = cmd.getVmId(); + final VMInstanceVO vm = findVmById(vmId); - validateForZone(vm.getDataCenterId()); + validateBackupForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); if (vm.getBackupOfferingId() == null) { @@ -591,10 +714,19 @@ public boolean createBackup(final Long vmId, Object job) throws ResourceAllocati throw new CloudRuntimeException("VM backup offering not found"); } + final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); + if (backupProvider == null) { + throw new CloudRuntimeException("VM backup provider not found for the offering"); + } + if (!offering.isUserDrivenBackupAllowed()) { throw new CloudRuntimeException("The assigned backup offering does not allow ad-hoc user backup"); } + if (!"nas".equals(offering.getProvider()) && cmd.getQuiesceVM() != null) { + throw new InvalidParameterValueException("Quiesce VM option is supported only for NAS backup provider"); + } + Long backupScheduleId = getBackupScheduleId(job); boolean isScheduledBackup = backupScheduleId != null; Account owner = accountManager.getAccount(vm.getAccountId()); @@ -631,26 +763,26 @@ public boolean createBackup(final Long vmId, Object job) throws ResourceAllocati vmId, ApiCommandResourceType.VirtualMachine.toString(), true, 0); - final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); - if (backupProvider != null) { - Pair result = backupProvider.takeBackup(vm); - if (!result.first()) { - throw new CloudRuntimeException("Failed to create VM backup"); - } - Backup backup = result.second(); - if (backup != null) { - BackupVO vmBackup = backupDao.findById(result.second().getId()); - vmBackup.setBackupScheduleId(backupScheduleId); - backupDao.update(vmBackup.getId(), vmBackup); - resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); - resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); - } - if (isScheduledBackup) { - deleteOldestBackupFromScheduleIfRequired(vmId, backupScheduleId); + Pair result = backupProvider.takeBackup(vm, cmd.getQuiesceVM()); + if (!result.first()) { + throw new CloudRuntimeException("Failed to create VM backup"); + } + Backup backup = result.second(); + if (backup != null) { + BackupVO vmBackup = backupDao.findById(result.second().getId()); + vmBackup.setBackupScheduleId(backupScheduleId); + if (cmd.getName() != null) { + vmBackup.setName(cmd.getName()); } - return true; + vmBackup.setDescription(cmd.getDescription()); + backupDao.update(vmBackup.getId(), vmBackup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); } - throw new CloudRuntimeException("Failed to create VM backup"); + if (isScheduledBackup) { + deleteOldestBackupFromScheduleIfRequired(vmId, backupScheduleId); + } + return true; } /** @@ -746,7 +878,9 @@ protected void deleteExcessBackups(List backups, int amountOfBackupsTo public Pair, Integer> listBackups(final ListBackupsCmd cmd) { final Long id = cmd.getId(); final Long vmId = cmd.getVmId(); + final String name = cmd.getName(); final Long zoneId = cmd.getZoneId(); + final Long backupOfferingId = cmd.getBackupOfferingId(); final Account caller = CallContext.current().getCallingAccount(); final String keyword = cmd.getKeyword(); List permittedAccounts = new ArrayList(); @@ -772,13 +906,16 @@ public Pair, Integer> listBackups(final ListBackupsCmd cmd) { sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); sb.and("idIN", sb.entity().getId(), SearchCriteria.Op.IN); sb.and("vmId", sb.entity().getVmId(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); sb.and("zoneId", sb.entity().getZoneId(), SearchCriteria.Op.EQ); + sb.and("backupOfferingId", sb.entity().getBackupOfferingId(), SearchCriteria.Op.EQ); if (keyword != null) { + sb.or().op("keywordName", sb.entity().getName(), SearchCriteria.Op.LIKE); SearchBuilder vmSearch = vmInstanceDao.createSearchBuilder(); - vmSearch.and("name", vmSearch.entity().getHostName(), SearchCriteria.Op.LIKE); - sb.groupBy(sb.entity().getId()); sb.join("vmSearch", vmSearch, sb.entity().getVmId(), vmSearch.entity().getId(), JoinBuilder.JoinType.INNER); + sb.or("vmSearch", "keywordVmName", vmSearch.entity().getHostName(), SearchCriteria.Op.LIKE); + sb.cp(); } SearchCriteria sc = sb.create(); @@ -792,12 +929,22 @@ public Pair, Integer> listBackups(final ListBackupsCmd cmd) { sc.setParameters("vmId", vmId); } + if (name != null) { + sc.setParameters("name", name); + } + if (zoneId != null) { sc.setParameters("zoneId", zoneId); } + if (backupOfferingId != null) { + sc.setParameters("backupOfferingId", backupOfferingId); + } + if (keyword != null) { - sc.setJoinParameters("vmSearch", "name", "%" + keyword + "%"); + String keywordMatch = "%" + keyword + "%"; + sc.setParameters("keywordName", keywordMatch); + sc.setParameters("keywordVmName", keywordMatch); } Pair, Integer> result = backupDao.searchAndCount(sc, searchFilter); @@ -839,11 +986,14 @@ public boolean restoreBackup(final Long backupId) { if (backup == null) { throw new CloudRuntimeException("Backup " + backupId + " does not exist"); } - validateForZone(backup.getZoneId()); + if (backup.getStatus() != Backup.Status.BackedUp) { + throw new CloudRuntimeException("Backup should be in BackedUp state"); + } + validateBackupForZone(backup.getZoneId()); final VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); - if (vm == null) { - throw new CloudRuntimeException("VM ID " + backup.getVmId() + " couldn't be found on existing or removed VMs"); + if (vm == null || VirtualMachine.State.Expunging.equals(vm.getState())) { + throw new CloudRuntimeException("The Instance from which the backup was taken could not be found."); } accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); @@ -978,6 +1128,217 @@ private Backup.VolumeInfo getVolumeInfo(List backedUpVolumes, return null; } + @Override + public void checkVmDisksSizeAgainstBackup(List vmDiskInfoList, Backup backup) { + List vmDiskInfoListFromBackup = getDataDiskInfoListFromBackup(backup); + int index = 0; + if (vmDiskInfoList.size() != vmDiskInfoListFromBackup.size()) { + throw new InvalidParameterValueException("Unable to create Instance from Backup " + + "as the backup has a different number of disks than the Instance."); + } + for (VmDiskInfo vmDiskInfo : vmDiskInfoList) { + if (index < vmDiskInfoListFromBackup.size()) { + if (vmDiskInfo.getSize() < vmDiskInfoListFromBackup.get(index).getSize()) { + throw new InvalidParameterValueException( + String.format("Instance volume size %d[GiB] cannot be less than the backed-up volume size %d[GiB].", + vmDiskInfo.getSize(), vmDiskInfoListFromBackup.get(index).getSize())); + } + } + index++; + } + } + + @Override + public VmDiskInfo getRootDiskInfoFromBackup(Backup backup) { + List volumes = backup.getBackedUpVolumes(); + VmDiskInfo rootDiskOffering = null; + if (volumes == null || volumes.isEmpty()) { + throw new CloudRuntimeException("Failed to get backed-up volumes info from backup"); + } + for (Backup.VolumeInfo volume : volumes) { + if (volume.getType() == Volume.Type.ROOT) { + DiskOfferingVO diskOffering = diskOfferingDao.findByUuid(volume.getDiskOfferingId()); + if (diskOffering == null) { + throw new CloudRuntimeException(String.format("Unable to find the root disk offering with uuid (%s) " + + "stored in backup. Please specify a valid root disk offering id while creating the instance", + volume.getDiskOfferingId())); + } + Long size = volume.getSize() / (1024 * 1024 * 1024); + rootDiskOffering = new VmDiskInfo(diskOffering, size, volume.getMinIops(), volume.getMaxIops()); + } + } + if (rootDiskOffering == null) { + throw new CloudRuntimeException("Failed to get the root disk in backed-up volumes info from backup"); + } + return rootDiskOffering; + } + + @Override + public List getDataDiskInfoListFromBackup(Backup backup) { + List vmDiskInfoList = new ArrayList<>(); + List volumes = backup.getBackedUpVolumes(); + if (volumes == null || volumes.isEmpty()) { + throw new CloudRuntimeException("Failed to get backed-up Volumes info from backup"); + } + for (Backup.VolumeInfo volume : volumes) { + if (volume.getType() == Volume.Type.DATADISK) { + DiskOfferingVO diskOffering = diskOfferingDao.findByUuid(volume.getDiskOfferingId()); + if (diskOffering == null || diskOffering.getState().equals(DiskOffering.State.Inactive)) { + throw new CloudRuntimeException("Unable to find the disk offering with uuid (" + volume.getDiskOfferingId() + ") stored in backup. " + + "Please specify a valid disk offering id while creating the instance"); + } + Long size = volume.getSize() / (1024 * 1024 * 1024); + vmDiskInfoList.add(new VmDiskInfo(diskOffering, size, volume.getMinIops(), volume.getMaxIops(), volume.getDeviceId())); + } + } + return vmDiskInfoList; + } + + @Override + public Map getIpToNetworkMapFromBackup(Backup backup, boolean preserveIps, List networkIds) + { + Map ipToNetworkMap = new LinkedHashMap(); + + String nicsJson = backup.getDetail(ApiConstants.NICS); + if (nicsJson == null) { + throw new CloudRuntimeException("Backup doesn't contain network information. " + + "Please specify at least one valid network while creating instance"); + } + + Type type = new TypeToken>>(){}.getType(); + List> nics = new Gson().fromJson(nicsJson, type); + + for (Map nic : nics) { + String networkUuid = nic.get(ApiConstants.NETWORK_ID); + if (networkUuid == null) { + throw new CloudRuntimeException("Backup doesn't contain network information. " + + "Please specify at least one valid network while creating instance"); + } + + Network network = networkDao.findByUuid(networkUuid); + if (network == null) { + throw new CloudRuntimeException("Unable to find network with the uuid " + networkUuid + " stored in backup. " + + "Please specify a valid network id while creating the instance"); + } + + Long networkId = network.getId(); + Network.IpAddresses ipAddresses = null; + + if (preserveIps) { + String ip = nic.get(ApiConstants.IP_ADDRESS); + String ipv6 = nic.get(ApiConstants.IP6_ADDRESS); + String mac = nic.get(ApiConstants.MAC_ADDRESS); + ipAddresses = networkService.getIpAddressesFromIps(ip, ipv6, mac); + } + + ipToNetworkMap.put(networkId, ipAddresses); + networkIds.add(networkId); + } + return ipToNetworkMap; + } + + @Override + public Boolean canCreateInstanceFromBackup(final Long backupId) { + final BackupVO backup = backupDao.findById(backupId); + BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); + if (offering == null) { + throw new CloudRuntimeException("Failed to find backup offering"); + } + final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); + return backupProvider.supportsInstanceFromBackup(); + } + + @Override + public boolean restoreBackupToVM(final Long backupId, final Long vmId) throws ResourceUnavailableException { + final BackupVO backup = backupDao.findById(backupId); + if (backup == null) { + throw new CloudRuntimeException("Backup " + backupId + " does not exist"); + } + if (backup.getStatus() != Backup.Status.BackedUp) { + throw new CloudRuntimeException("Backup should be in BackedUp state"); + } + validateBackupForZone(backup.getZoneId()); + + VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(vmId); + if (vm == null) { + throw new CloudRuntimeException("Instance with ID " + backup.getVmId() + " couldn't be found."); + } + accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); + + if (vm.getRemoved() != null) { + throw new CloudRuntimeException("Instance with ID " + backup.getVmId() + " couldn't be found."); + } + if (!vm.getState().equals(VirtualMachine.State.Stopped)) { + throw new CloudRuntimeException("The VM should be in stopped state"); + } + + List backupVolumes = backup.getBackedUpVolumes(); + if (backupVolumes == null) { + throw new CloudRuntimeException("Backed up volumes info not found in the backup"); + } + + List vmVolumes = volumeDao.findByInstance(vmId); + if (vmVolumes.size() != backupVolumes.size()) { + throw new CloudRuntimeException("Unable to create Instance from backup as the backup has a different number of disks than the Instance"); + } + + int index = 0; + for (VolumeVO vmVolume: vmVolumes) { + Backup.VolumeInfo backupVolume = backupVolumes.get(index); + if (vmVolume.getSize() < backupVolume.getSize()) { + throw new CloudRuntimeException(String.format( + "Instance volume size %d[GiB] for volume (%s) is less than the backed-up volume size %d[GiB] for backed-up volume (%s).", + vmVolume.getSize(), vmVolume.getUuid(), backupVolume.getSize(), backupVolume.getUuid())); + } + index++; + } + + BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); + if (offering == null) { + throw new CloudRuntimeException("Failed to find backup offering"); + } + final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); + if (!backupProvider.supportsInstanceFromBackup()) { + throw new CloudRuntimeException("Create instance from backup is not supported by the " + offering.getProvider() + " provider."); + } + + String backupDetailsInMessage = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(backup, "uuid", "externalId", "newVMId", "type", "status", "date"); + Long eventId = null; + try { + updateVmState(vm, VirtualMachine.Event.RestoringRequested, VirtualMachine.State.Restoring); + updateVolumeState(vm, Volume.Event.RestoreRequested, Volume.State.Restoring); + eventId = ActionEventUtils.onStartedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventTypes.EVENT_VM_CREATE_FROM_BACKUP, + String.format("Creating Instance %s from backup %s", vm.getInstanceName(), backup.getUuid()), + vm.getId(), ApiCommandResourceType.VirtualMachine.toString(), + true, 0); + + String host = null; + String dataStore = null; + if (!"nas".equals(offering.getProvider())) { + Pair restoreInfo = getRestoreVolumeHostAndDatastore(vm); + host = restoreInfo.first().getPrivateIpAddress(); + dataStore = restoreInfo.second().getUuid(); + } + if (!backupProvider.restoreBackupToVM(vm, backup, host, dataStore)) { + throw new CloudRuntimeException(String.format("Error restoring backup [%s] to VM %s.", backupDetailsInMessage, vm.getUuid())); + } + } catch (Exception e) { + updateVolumeState(vm, Volume.Event.RestoreFailed, Volume.State.Ready); + updateVmState(vm, VirtualMachine.Event.RestoringFailed, VirtualMachine.State.Stopped); + logger.error(String.format("Failed to create Instance [%s] from backup [%s] due to: [%s].", vm.getInstanceName(), backupDetailsInMessage, e.getMessage()), e); + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_ERROR, EventTypes.EVENT_VM_CREATE_FROM_BACKUP, + String.format("Failed to create Instance %s from backup %s", vm.getInstanceName(), backup.getUuid()), + vm.getId(), ApiCommandResourceType.VirtualMachine.toString(), eventId); + throw new CloudRuntimeException(String.format("Error while creating Instance [%s] from backup [%s].", vm.getUuid(), backupDetailsInMessage)); + } + updateVolumeState(vm, Volume.Event.RestoreSucceeded, Volume.State.Ready); + updateVmState(vm, VirtualMachine.Event.RestoringSuccess, VirtualMachine.State.Stopped); + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_CREATE_FROM_BACKUP, + String.format("Successfully created Instance %s from backup %s", vm.getInstanceName(), backup.getUuid()), + vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),eventId); + return true; + } + @Override @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_RESTORE, eventDescription = "restoring VM from backup", async = true) public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid, final Long backupId, final Long vmId) throws Exception { @@ -988,7 +1349,10 @@ public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid, if (backup == null) { throw new CloudRuntimeException("Provided backup not found"); } - validateForZone(backup.getZoneId()); + if (backup.getStatus() != Backup.Status.BackedUp) { + throw new CloudRuntimeException("Backup should be in BackedUp state"); + } + validateBackupForZone(backup.getZoneId()); final VMInstanceVO vm = findVmById(vmId); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); @@ -1001,11 +1365,22 @@ public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid, throw new CloudRuntimeException("Cross zone backup restoration of volume is not allowed"); } - final VMInstanceVO vmFromBackup = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); - if (vmFromBackup == null) { - throw new CloudRuntimeException("VM reference for the provided VM backup not found"); + List volumeInfoList = backup.getBackedUpVolumes(); + if (volumeInfoList == null) { + final VMInstanceVO vmFromBackup = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); + if (vmFromBackup == null) { + throw new CloudRuntimeException("VM reference for the provided VM backup not found"); + } else if (vmFromBackup == null || vmFromBackup.getBackupVolumeList() == null) { + throw new CloudRuntimeException("Volumes metadata not found in the backup"); + } + volumeInfoList = vm.getBackupVolumeList(); } - accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vmFromBackup); + Backup.VolumeInfo backupVolumeInfo = getVolumeInfo(volumeInfoList, backedUpVolumeUuid); + if (backupVolumeInfo == null) { + throw new CloudRuntimeException("Failed to find volume with Id " + backedUpVolumeUuid + " in the backed-up volumes metadata"); + } + + accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); final BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); if (offering == null) { throw new CloudRuntimeException("Failed to find VM backup offering"); @@ -1014,7 +1389,7 @@ public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid, BackupProvider backupProvider = getBackupProvider(offering.getProvider()); VolumeVO backedUpVolume = volumeDao.findByUuid(backedUpVolumeUuid); Pair restoreInfo; - if (!"nas".equals(offering.getProvider())) { + if (!"nas".equals(offering.getProvider()) || (backedUpVolume == null)) { restoreInfo = getRestoreVolumeHostAndDatastore(vm); } else { restoreInfo = getRestoreVolumeHostAndDatastoreForNas(vm, backedUpVolume); @@ -1031,36 +1406,36 @@ public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid, String[] hostPossibleValues = {host.getPrivateIpAddress(), host.getName()}; String[] datastoresPossibleValues = {datastore.getUuid(), datastore.getName()}; - Pair result = restoreBackedUpVolume(backedUpVolumeUuid, backup, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + Pair result = restoreBackedUpVolume(backupVolumeInfo, backup, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); if (BooleanUtils.isFalse(result.first())) { throw new CloudRuntimeException(String.format("Error restoring volume [%s] of VM [%s] to host [%s] using backup provider [%s] due to: [%s].", backedUpVolumeUuid, vm.getUuid(), host.getUuid(), backupProvider.getName(), result.second())); } - if (!attachVolumeToVM(vm.getDataCenterId(), result.second(), vmFromBackup.getBackupVolumeList(), + if (!attachVolumeToVM(vm.getDataCenterId(), result.second(), backupVolumeInfo, backedUpVolumeUuid, vm, datastore.getUuid(), backup)) { - throw new CloudRuntimeException(String.format("Error attaching volume [%s] to VM [%s]." + backedUpVolumeUuid, vm.getUuid())); + throw new CloudRuntimeException(String.format("Error attaching volume [%s] to VM [%s].", backedUpVolumeUuid, vm.getUuid())); } return true; } - protected Pair restoreBackedUpVolume(final String backedUpVolumeUuid, final BackupVO backup, BackupProvider backupProvider, String[] hostPossibleValues, - String[] datastoresPossibleValues, VMInstanceVO vm) { + protected Pair restoreBackedUpVolume(final Backup.VolumeInfo backupVolumeInfo, final BackupVO backup, + BackupProvider backupProvider, String[] hostPossibleValues, String[] datastoresPossibleValues, VMInstanceVO vm) { Pair result = new Pair<>(false, ""); for (String hostData : hostPossibleValues) { for (String datastoreData : datastoresPossibleValues) { logger.debug(String.format("Trying to restore volume [UUID: %s], using host [%s] and datastore [%s].", - backedUpVolumeUuid, hostData, datastoreData)); + backupVolumeInfo.getUuid(), hostData, datastoreData)); try { - result = backupProvider.restoreBackedUpVolume(backup, backedUpVolumeUuid, hostData, datastoreData, new Pair<>(vm.getName(), vm.getState())); + result = backupProvider.restoreBackedUpVolume(backup, backupVolumeInfo, hostData, datastoreData, new Pair<>(vm.getName(), vm.getState())); if (BooleanUtils.isTrue(result.first())) { return result; } } catch (Exception e) { logger.debug(String.format("Failed to restore volume [UUID: %s], using host [%s] and datastore [%s] due to: [%s].", - backedUpVolumeUuid, hostData, datastoreData, e.getMessage()), e); + backupVolumeInfo.getUuid(), hostData, datastoreData, e.getMessage()), e); } } } @@ -1074,23 +1449,32 @@ public boolean deleteBackup(final Long backupId, final Boolean forced) { if (backup == null) { throw new CloudRuntimeException("Backup " + backupId + " does not exist"); } + final Long vmId = backup.getVmId(); final VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(vmId); if (vm == null) { - throw new CloudRuntimeException("VM " + vmId + " does not exist"); + logger.warn("Instance {} not found for backup {} during delete backup", vmId, backup.toString()); } - validateForZone(vm.getDataCenterId()); - accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); + logger.debug("Deleting backup {} belonging to instance {}", backup.toString(), vmId); + + validateBackupForZone(backup.getZoneId()); + accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm == null ? backup : vm); final BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); if (offering == null) { throw new CloudRuntimeException(String.format("Backup offering with ID [%s] does not exist.", backup.getBackupOfferingId())); } - final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); + final BackupProvider backupProvider = getBackupProvider(backup.getZoneId()); boolean result = backupProvider.deleteBackup(backup, forced); if (result) { - resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); - resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); - return backupDao.remove(backup.getId()); + resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup); + Long backupSize = backup.getSize() != null ? backup.getSize() : 0L; + resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup_storage, backupSize); + if (backupDao.remove(backup.getId())) { + checkAndGenerateUsageForLastBackupDeletedAfterOfferingRemove(vm, backup); + return true; + } else { + return false; + } } throw new CloudRuntimeException("Failed to delete the backup"); } @@ -1135,24 +1519,32 @@ private HostVO getFirstHostFromStoragePool(StoragePoolVO storagePoolVO) { /** * Attach volume to VM */ - private boolean attachVolumeToVM(Long zoneId, String restoredVolumeLocation, List backedUpVolumes, + private boolean attachVolumeToVM(Long zoneId, String restoredVolumeLocation, Backup.VolumeInfo backupVolumeInfo, String volumeUuid, VMInstanceVO vm, String datastoreUuid, Backup backup) throws Exception { HypervisorGuru guru = hypervisorGuruManager.getGuru(vm.getHypervisorType()); - Backup.VolumeInfo volumeInfo = getVolumeInfo(backedUpVolumes, volumeUuid); - if (volumeInfo == null) { - throw new CloudRuntimeException("Failed to find volume in the backedup volumes of ID " + volumeUuid); - } - volumeInfo.setType(Volume.Type.DATADISK); + backupVolumeInfo.setType(Volume.Type.DATADISK); logger.debug("Attaching the restored volume to VM {}", vm); StoragePoolVO pool = primaryDataStoreDao.findByUuid(datastoreUuid); try { - return guru.attachRestoredVolumeToVirtualMachine(zoneId, restoredVolumeLocation, volumeInfo, vm, pool.getId(), backup); + return guru.attachRestoredVolumeToVirtualMachine(zoneId, restoredVolumeLocation, backupVolumeInfo, vm, pool.getId(), backup); } catch (Exception e) { throw new CloudRuntimeException("Error attach restored volume to VM " + vm.getUuid() + " due to: " + e.getMessage()); } } + private void checkAndGenerateUsageForLastBackupDeletedAfterOfferingRemove(VirtualMachine vm, Backup backup) { + if (vm != null && + (vm.getBackupOfferingId() == null || vm.getBackupOfferingId() != backup.getBackupOfferingId())) { + List backups = backupDao.listByVmIdAndOffering(vm.getDataCenterId(), vm.getId(), backup.getBackupOfferingId()); + if (backups.size() == 0) { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVED_AND_BACKUPS_DELETED, vm.getAccountId(), + vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(), + backup.getBackupOfferingId(), null, null, Backup.class.getSimpleName(), vm.getUuid()); + } + } + } + @Override public boolean configure(String name, Map params) throws ConfigurationException { super.configure(name, params); @@ -1164,7 +1556,8 @@ public boolean isDisabled(final Long zoneId) { return !(BackupFrameworkEnabled.value() && BackupFrameworkEnabled.valueIn(zoneId)); } - private void validateForZone(final Long zoneId) { + @Override + public void validateBackupForZone(final Long zoneId) { if (zoneId == null || isDisabled(zoneId)) { throw new CloudRuntimeException("Backup and Recovery feature is disabled for the zone"); } @@ -1222,6 +1615,8 @@ public List> getCommands() { cmdList.add(AddBackupRepositoryCmd.class); cmdList.add(DeleteBackupRepositoryCmd.class); cmdList.add(ListBackupRepositoriesCmd.class); + cmdList.add(CreateVMFromBackupCmd.class); + cmdList.add(CreateVMFromBackupCmdByAdmin.class); return cmdList; } @@ -1242,7 +1637,8 @@ public ConfigKey[] getConfigKeys() { DefaultMaxProjectBackups, DefaultMaxProjectBackupStorage, DefaultMaxDomainBackups, - DefaultMaxDomainBackupStorage + DefaultMaxDomainBackupStorage, + BackupStorageCapacityThreshold }; } @@ -1335,6 +1731,7 @@ public void scheduleBackups() { for (final BackupScheduleVO backupSchedule: backupsToBeExecuted) { final Long backupScheduleId = backupSchedule.getId(); final Long vmId = backupSchedule.getVmId(); + final Boolean quiesceVm = backupSchedule.getQuiesceVM(); final VMInstanceVO vm = vmInstanceDao.findById(vmId); if (vm == null || vm.getBackupOfferingId() == null) { @@ -1376,6 +1773,9 @@ public void scheduleBackups() { final Map params = new HashMap(); params.put(ApiConstants.VIRTUAL_MACHINE_ID, "" + vmId); params.put(ApiConstants.SCHEDULE_ID, String.valueOf(backupScheduleId)); + if (quiesceVm != null) { + params.put(ApiConstants.QUIESCE_VM, "" + quiesceVm.toString()); + } params.put("ctxUserId", "1"); params.put("ctxAccountId", "" + vm.getAccountId()); params.put("ctxStartEventId", String.valueOf(eventId)); @@ -1469,42 +1869,93 @@ protected void runInContext() { continue; } - List vms = vmInstanceDao.listByZoneWithBackups(dataCenter.getId(), null); - if (vms == null || vms.isEmpty()) { - logger.debug("Can't find any VM to sync backups in zone {}", dataCenter); - continue; - } + backupProvider.syncBackupStorageStats(dataCenter.getId()); + + syncOutOfBandBackups(backupProvider, dataCenter); - final Map metrics = backupProvider.getBackupMetrics(dataCenter.getId(), new ArrayList<>(vms)); - syncBackupMetrics(backupProvider, metrics); + updateBackupUsageRecords(backupProvider, dataCenter); } } catch (final Throwable t) { logger.error(String.format("Error trying to run backup-sync background task due to: [%s].", t.getMessage()), t); } } - /** - * Tries to sync the VM backups. If one backup synchronization fails, only this VM backups are skipped, and the entire process does not stop. - */ - private void syncBackupMetrics(final BackupProvider backupProvider, final Map metrics) { - for (final VirtualMachine vm : metrics.keySet()) { - tryToSyncVMBackups(backupProvider, metrics, vm); + private void syncOutOfBandBackups(final BackupProvider backupProvider, DataCenter dataCenter) { + List vms = vmInstanceDao.listByZoneAndBackupOffering(dataCenter.getId(), null); + if (vms == null || vms.isEmpty()) { + logger.debug("Can't find any VM to sync backups in zone {}", dataCenter); + return; + } + backupProvider.syncBackupMetrics(dataCenter.getId()); + for (final VMInstanceVO vm : vms) { + try { + logger.debug(String.format("Trying to sync backups of VM [%s] using backup provider [%s].", vm, backupProvider.getName())); + // Sync out-of-band backups + syncBackups(backupProvider, vm); + } catch (final Exception e) { + logger.error("Failed to sync backup usage metrics and out-of-band backups of VM [{}] due to: [{}].", vm, e.getMessage(), e); + } } } - private Backup checkAndUpdateIfBackupEntryExistsForRestorePoint(Backup.RestorePoint restorePoint, List backupsInDb, VirtualMachine vm, Backup.Metric metric) { + private void updateBackupUsageRecords(final BackupProvider backupProvider, DataCenter dataCenter) { + List vmIdsWithBackups = backupDao.listVmIdsWithBackupsInZone(dataCenter.getId()); + List vmsWithBackups; + if (vmIdsWithBackups.size() == 0) { + vmsWithBackups = new ArrayList<>(); + } else { + vmsWithBackups = vmInstanceDao.listByIdsIncludingRemoved(vmIdsWithBackups); + } + List vmsWithBackupOffering = vmInstanceDao.listByZoneAndBackupOffering(dataCenter.getId(), null); //should return including removed + Set vms = Stream.concat(vmsWithBackups.stream(), vmsWithBackupOffering.stream()) .collect(Collectors.toSet()); + + for (final VirtualMachine vm : vms) { + + Map> backupOfferingToSizeMap = new HashMap<>(); + for (final Backup backup: backupDao.listByVmId(null, vm.getId())) { + Long backupSize = 0L; + Long backupProtectedSize = 0L; + if (Objects.nonNull(backup.getSize())) { + backupSize = backup.getSize(); + } + if (Objects.nonNull(backup.getProtectedSize())) { + backupProtectedSize = backup.getProtectedSize(); + } + Long offeringId = backup.getBackupOfferingId(); + if (backupOfferingToSizeMap.containsKey(offeringId)) { + Pair sizes = backupOfferingToSizeMap.get(offeringId); + sizes.set(sizes.first() + backupSize, sizes.second() + backupProtectedSize); + } else { + backupOfferingToSizeMap.put(offeringId, new Pair<>(backupSize, backupProtectedSize)); + } + } + + for (final Map.Entry> entry : backupOfferingToSizeMap.entrySet()) { + Long offeringId = entry.getKey(); + Pair sizes = entry.getValue(); + Long backupSize = sizes.first(); + Long protectedSize = sizes.second(); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_USAGE_METRIC, vm.getAccountId(), + vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(), + offeringId, null, backupSize, protectedSize, + Backup.class.getSimpleName(), vm.getUuid()); + } + } + } + + private Backup checkAndUpdateIfBackupEntryExistsForRestorePoint(Backup.RestorePoint restorePoint, List backupsInDb, VirtualMachine vm) { for (final Backup backupInDb : backupsInDb) { logger.debug(String.format("Checking if Backup %s with external ID %s for VM %s is valid", backupsInDb, backupInDb.getName(), vm)); if (restorePoint.getId().equals(backupInDb.getExternalId())) { - logger.debug(String.format("Found Backup %s in both Database and Networker", backupInDb)); - if (metric != null) { + logger.debug(String.format("Found Backup %s in both Database and Provider", backupInDb)); + if (restorePoint.getDataSize() != null && restorePoint.getBackupSize() != null) { logger.debug(String.format("Update backup [%s] from [size: %s, protected size: %s] to [size: %s, protected size: %s].", - backupInDb, backupInDb.getSize(), backupInDb.getProtectedSize(), metric.getBackupSize(), metric.getDataSize())); + backupInDb, backupInDb.getSize(), backupInDb.getProtectedSize(), restorePoint.getBackupSize(), restorePoint.getDataSize())); - resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backupInDb.getSize()); - ((BackupVO) backupInDb).setSize(metric.getBackupSize()); - ((BackupVO) backupInDb).setProtectedSize(metric.getDataSize()); - resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backupInDb.getSize()); + resourceLimitMgr.decrementResourceCount(backupInDb.getAccountId(), Resource.ResourceType.backup_storage, backupInDb.getSize()); + ((BackupVO) backupInDb).setSize(restorePoint.getBackupSize()); + ((BackupVO) backupInDb).setProtectedSize(restorePoint.getDataSize()); + resourceLimitMgr.incrementResourceCount(backupInDb.getAccountId(), Resource.ResourceType.backup_storage, backupInDb.getSize()); backupDao.update(backupInDb.getId(), ((BackupVO) backupInDb)); } @@ -1514,7 +1965,22 @@ private Backup checkAndUpdateIfBackupEntryExistsForRestorePoint(Backup.RestorePo return null; } - private void syncBackups(BackupProvider backupProvider, VirtualMachine vm, Backup.Metric metric) { + private void processRemoveList(List removeList, VirtualMachine vm) { + for (final Long backupIdToRemove : removeList) { + logger.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove)); + Backup backup = backupDao.findById(backupIdToRemove); + resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); + boolean result = backupDao.remove(backupIdToRemove); + if (result) { + checkAndGenerateUsageForLastBackupDeletedAfterOfferingRemove(vm, backup); + } else { + logger.error("Failed to remove backup db entry ith ID: {} during sync backups", backupIdToRemove); + } + } + } + + private void syncBackups(BackupProvider backupProvider, VirtualMachine vm) { Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) { @@ -1527,14 +1993,14 @@ public void doInTransactionWithoutResult(TransactionStatus status) { final List removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList()); for (final Backup.RestorePoint restorePoint : restorePoints) { if (!(restorePoint.getId() == null || restorePoint.getType() == null || restorePoint.getCreated() == null)) { - Backup existingBackupEntry = checkAndUpdateIfBackupEntryExistsForRestorePoint(restorePoint, backupsInDb, vm, metric); + Backup existingBackupEntry = checkAndUpdateIfBackupEntryExistsForRestorePoint(restorePoint, backupsInDb, vm); if (existingBackupEntry != null) { removeList.remove(existingBackupEntry.getId()); continue; } } - Backup backup = backupProvider.createNewBackupEntryForRestorePoint(restorePoint, vm, metric); + Backup backup = backupProvider.createNewBackupEntryForRestorePoint(restorePoint, vm); if (backup != null) { logger.warn("Added backup found in provider [" + backup + "]"); resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); @@ -1549,35 +2015,11 @@ public void doInTransactionWithoutResult(TransactionStatus status) { vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); } } - for (final Long backupIdToRemove : removeList) { - logger.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove)); - Backup backup = backupDao.findById(backupIdToRemove); - resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); - resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); - backupDao.remove(backupIdToRemove); - } + processRemoveList(removeList, vm); } }); } - private void tryToSyncVMBackups(BackupProvider backupProvider, Map metrics, VirtualMachine vm) { - try { - final Backup.Metric metric = metrics.get(vm); - if (metric != null) { - logger.debug(String.format("Trying to sync backups of VM [%s] using backup provider [%s].", vm, backupProvider.getName())); - // Sync out-of-band backups - syncBackups(backupProvider, vm, metric); - // Emit a usage event, update usage metric for the VM by the usage server - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_USAGE_METRIC, vm.getAccountId(), - vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(), - vm.getBackupOfferingId(), null, metric.getBackupSize(), metric.getDataSize(), - Backup.class.getSimpleName(), vm.getUuid()); - } - } catch (final Exception e) { - logger.error("Failed to sync backup usage metrics and out-of-band backups of VM [{}] due to: [{}].", vm, e.getMessage(), e); - } - } - @Override public Long getDelay() { return BackupSyncPollingInterval.value() * 1000L; @@ -1628,4 +2070,124 @@ public BackupOffering updateBackupOffering(UpdateBackupOfferingCmd updateBackupO return response; } + Map getDetailsFromBackupDetails(Long backupId) { + Map details = backupDetailsDao.listDetailsKeyPairs(backupId, true); + if (details == null) { + return null; + } + if (details.containsKey(ApiConstants.TEMPLATE_ID)) { + VirtualMachineTemplate template = vmTemplateDao.findByUuid(details.get(ApiConstants.TEMPLATE_ID)); + if (template != null) { + details.put(ApiConstants.TEMPLATE_ID, template.getUuid()); + details.put(ApiConstants.TEMPLATE_NAME, template.getName()); + details.put(ApiConstants.IS_ISO, String.valueOf(template.getFormat().equals(Storage.ImageFormat.ISO))); + } + } + if (details.containsKey(ApiConstants.SERVICE_OFFERING_ID)) { + ServiceOffering serviceOffering = serviceOfferingDao.findByUuid(details.get(ApiConstants.SERVICE_OFFERING_ID)); + if (serviceOffering != null) { + details.put(ApiConstants.SERVICE_OFFERING_ID, serviceOffering.getUuid()); + details.put(ApiConstants.SERVICE_OFFERING_NAME, serviceOffering.getName()); + } + } + if (details.containsKey(ApiConstants.NICS)) { + Type type = new TypeToken>>() {}.getType(); + List> nics = new Gson().fromJson(details.get(ApiConstants.NICS), type); + + for (Map nic : nics) { + String networkUuid = nic.get(ApiConstants.NETWORK_ID); + if (networkUuid != null) { + Network network = networkDao.findByUuid(networkUuid); + if (network != null) { + nic.put(ApiConstants.NETWORK_NAME, network.getName()); + } + } + } + details.put(ApiConstants.NICS, new Gson().toJson(nics)); + } + return details; + } + + @Override + public BackupResponse createBackupResponse(Backup backup, Boolean listVmDetails) { + VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); + AccountVO account = accountDao.findByIdIncludingRemoved(backup.getAccountId()); + DomainVO domain = domainDao.findByIdIncludingRemoved(backup.getDomainId()); + DataCenterVO zone = dataCenterDao.findByIdIncludingRemoved(backup.getZoneId()); + Long offeringId = backup.getBackupOfferingId(); + BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(offeringId); + + BackupResponse response = new BackupResponse(); + response.setId(backup.getUuid()); + response.setName(backup.getName()); + response.setDescription(backup.getDescription()); + response.setVmName(vm.getHostName()); + response.setVmId(vm.getUuid()); + if (vm.getBackupOfferingId() == null || vm.getBackupOfferingId() != backup.getBackupOfferingId()) { + response.setVmOfferingRemoved(true); + } + response.setExternalId(backup.getExternalId()); + response.setType(backup.getType()); + response.setDate(backup.getDate()); + response.setSize(backup.getSize()); + response.setProtectedSize(backup.getProtectedSize()); + response.setStatus(backup.getStatus()); + response.setIntervalType("MANUAL"); + if (backup.getBackupScheduleId() != null) { + BackupScheduleVO scheduleVO = backupScheduleDao.findById(backup.getBackupScheduleId()); + if (scheduleVO != null) { + response.setIntervalType(scheduleVO.getScheduleType().toString()); + } + } + // ACS 4.20: For backups taken prior this release the backup.backed_volumes column would be empty hence use vm_instance.backup_volumes + String backedUpVolumes; + if (Objects.isNull(backup.getBackedUpVolumes())) { + backedUpVolumes = new Gson().toJson(vm.getBackupVolumeList().toArray(), Backup.VolumeInfo[].class); + } else { + backedUpVolumes = new Gson().toJson(backup.getBackedUpVolumes().toArray(), Backup.VolumeInfo[].class); + } + response.setVolumes(backedUpVolumes); + response.setBackupOfferingId(offering.getUuid()); + response.setBackupOffering(offering.getName()); + response.setAccountId(account.getUuid()); + response.setAccount(account.getAccountName()); + response.setDomainId(domain.getUuid()); + response.setDomain(domain.getName()); + response.setZoneId(zone.getUuid()); + response.setZone(zone.getName()); + + if (Boolean.TRUE.equals(listVmDetails)) { + Map vmDetails = new HashMap<>(); + vmDetails.put(ApiConstants.HYPERVISOR, vm.getHypervisorType().toString()); + Map details = getDetailsFromBackupDetails(backup.getId()); + vmDetails.putAll(details); + response.setVmDetails(vmDetails); + } + + response.setObjectName("backup"); + return response; + } + + @Override + public CapacityVO getBackupStorageUsedStats(Long zoneId) { + final BackupProvider backupProvider = getBackupProvider(zoneId); + Pair backupUsage = backupProvider.getBackupStorageStats(zoneId); + return new CapacityVO(null, zoneId, null, null, backupUsage.first(), backupUsage.second(), Capacity.CAPACITY_TYPE_BACKUP_STORAGE); + } + + @Override + public void checkAndRemoveBackupOfferingBeforeExpunge(VirtualMachine vm) { + if (vm.getBackupOfferingId() == null) { + return; + } + List backupsForVm = backupDao.listByVmIdAndOffering(vm.getDataCenterId(), vm.getId(), vm.getBackupOfferingId()); + if (org.apache.commons.collections.CollectionUtils.isEmpty(backupsForVm)) { + removeVMFromBackupOffering(vm.getId(), true); + } else { + throw new CloudRuntimeException(String.format("This Instance [uuid: %s, name: %s] has a " + + "Backup Offering [id: %s, external id: %s] with %s backups. Please, remove the backup offering " + + "before proceeding to VM exclusion!", vm.getUuid(), vm.getInstanceName(), vm.getBackupOfferingId(), + vm.getBackupExternalId(), backupsForVm.size())); + } + } } diff --git a/server/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManagerImpl.java b/server/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManagerImpl.java index 38ab45f20b44..0ab81a4c1d6d 100644 --- a/server/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManagerImpl.java @@ -112,7 +112,7 @@ public class ConsoleAccessManagerImpl extends ManagerBase implements ConsoleAcce protected Logger logger = LogManager.getLogger(ConsoleAccessManagerImpl.class); private static final List unsupportedConsoleVMState = Arrays.asList( - VirtualMachine.State.Stopped, VirtualMachine.State.Error, VirtualMachine.State.Destroyed + VirtualMachine.State.Stopped, VirtualMachine.State.Restoring, VirtualMachine.State.Error, VirtualMachine.State.Destroyed ); @Override diff --git a/server/src/main/java/org/apache/cloudstack/resource/ResourceCleanupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/resource/ResourceCleanupServiceImpl.java index 298753a6c4f6..b752a4c384f3 100644 --- a/server/src/main/java/org/apache/cloudstack/resource/ResourceCleanupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/resource/ResourceCleanupServiceImpl.java @@ -36,6 +36,8 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.resource.PurgeExpungedResourcesCmd; +import org.apache.cloudstack.backup.BackupVO; +import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; @@ -158,6 +160,8 @@ public class ResourceCleanupServiceImpl extends ManagerBase implements ResourceC ManagementServerHostDao managementServerHostDao; @Inject ServiceOfferingDetailsDao serviceOfferingDetailsDao; + @Inject + BackupDao backupDao; private ScheduledExecutorService expungedResourcesCleanupExecutor; private ExecutorService purgeExpungedResourcesJobExecutor; @@ -300,7 +304,7 @@ protected HashSet getVmIdsWithActiveVolumeSnapshots(List vmIds) { .collect(Collectors.toCollection(HashSet::new)); } - protected Pair, List> getFilteredVmIdsForSnapshots(List vmIds) { + protected Pair, List> getFilteredVmIdsForSnapshotsAndBackups(List vmIds) { HashSet currentSkippedVmIds = new HashSet<>(); List activeSnapshots = vmSnapshotDao.searchByVms(vmIds); if (CollectionUtils.isNotEmpty(activeSnapshots)) { @@ -320,20 +324,33 @@ protected Pair, List> getFilteredVmIdsForSnapshots(List v } currentSkippedVmIds.addAll(vmIdsWithActiveVolumeSnapshots); } + + List backups = backupDao.searchByVmIds(vmIds); + if (CollectionUtils.isNotEmpty(backups)) { + HashSet vmIdsWithBackups = backups.stream().map(BackupVO::getVmId) + .collect(Collectors.toCollection(HashSet::new)); + if (logger.isDebugEnabled()) { + logger.debug(String.format("Skipping purging VMs with IDs %s as they have backups", + StringUtils.join(vmIdsWithBackups))); + } + currentSkippedVmIds.addAll(vmIdsWithBackups); + } + if (CollectionUtils.isNotEmpty(currentSkippedVmIds)) { vmIds.removeAll(currentSkippedVmIds); } + return new Pair<>(vmIds, new ArrayList<>(currentSkippedVmIds)); } - protected Pair, List> getVmIdsWithNoActiveSnapshots(final Date startDate, final Date endDate, - final Long batchSize, final List skippedVmIds) { + protected Pair, List> getVmIdsWithNoActiveSnapshotsAndBackups(final Date startDate, final Date endDate, + final Long batchSize, final List skippedVmIds) { List vms = vmInstanceDao.searchRemovedByRemoveDate(startDate, endDate, batchSize, skippedVmIds); if (CollectionUtils.isEmpty(vms)) { return new Pair<>(new ArrayList<>(), new ArrayList<>()); } List vmIds = vms.stream().map(VMInstanceVO::getId).collect(Collectors.toList()); - return getFilteredVmIdsForSnapshots(vmIds); + return getFilteredVmIdsForSnapshotsAndBackups(vmIds); } protected long purgeVMEntities(final Long batchSize, final Date startDate, final Date endDate) { @@ -344,7 +361,7 @@ protected long purgeVMEntities(final Long batchSize, final Date startDate, final List skippedVmIds = new ArrayList<>(); do { Pair, List> allVmIds = - getVmIdsWithNoActiveSnapshots(startDate, endDate, batchSize, skippedVmIds); + getVmIdsWithNoActiveSnapshotsAndBackups(startDate, endDate, batchSize, skippedVmIds); List vmIds = allVmIds.first(); List currentSkippedVmIds = allVmIds.second(); count = vmIds.size() + currentSkippedVmIds.size(); @@ -364,7 +381,7 @@ protected boolean purgeVMEntity(final long vmId) { final Long batchSize = ExpungedResourcesPurgeBatchSize.value().longValue(); List vmIds = new ArrayList<>(); vmIds.add(vmId); - Pair, List> allVmIds = getFilteredVmIdsForSnapshots(vmIds); + Pair, List> allVmIds = getFilteredVmIdsForSnapshotsAndBackups(vmIds); if (CollectionUtils.isEmpty(allVmIds.first())) { return false; } diff --git a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java index ea3361507ca7..5a18f16fd724 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java @@ -173,6 +173,14 @@ public Bucket createBucket(CreateBucketCmd cmd) { if (cmd.getQuota() != null) { objectStore.setQuota(bucketTO, cmd.getQuota()); resourceLimitManager.incrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, (cmd.getQuota() * Resource.ResourceType.bytesToGiB)); + if (objectStoreVO.getTotalSize() != null && objectStoreVO.getTotalSize() != 0 && objectStoreVO.getAllocatedSize() != null) { + Long allocatedSize = objectStoreVO.getAllocatedSize() / Resource.ResourceType.bytesToGiB; + Long totalSize = objectStoreVO.getTotalSize() / Resource.ResourceType.bytesToGiB; + if (cmd.getQuota() + allocatedSize > totalSize) { + logger.error("Object store {}'s allocated size has reached the total size limit of {}GiB.", objectStoreVO.getName(), totalSize); + throw new CloudRuntimeException("Not enough space in object store to create the bucket"); + } + } } if (cmd.getPolicy() != null) { @@ -181,6 +189,9 @@ public Bucket createBucket(CreateBucketCmd cmd) { bucket.setState(Bucket.State.Created); _bucketDao.update(bucket.getId(), bucket); + if (cmd.getQuota() != null) { + _objectStoreDao.updateAllocatedSize(objectStoreVO, cmd.getQuota() * Resource.ResourceType.bytesToGiB); + } } catch (Exception e) { logger.debug("Failed to create bucket with name: "+bucket.getName(), e); if(bucketCreated) { @@ -205,7 +216,10 @@ public boolean deleteBucket(long bucketId, Account caller) { ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); if (objectStore.deleteBucket(bucketTO)) { resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.bucket); - resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, (bucket.getQuota() * Resource.ResourceType.bytesToGiB)); + if (bucket.getQuota() != null) { + resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, (bucket.getQuota() * Resource.ResourceType.bytesToGiB)); + _objectStoreDao.updateAllocatedSize(objectStoreVO, -(bucket.getQuota() * Resource.ResourceType.bytesToGiB)); + } return _bucketDao.remove(bucketId); } return false; @@ -265,6 +279,7 @@ public boolean updateBucket(UpdateBucketCmd cmd, Account caller) throws Resource } else { resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, ((-quotaDelta) * Resource.ResourceType.bytesToGiB)); } + _objectStoreDao.updateAllocatedSize(objectStoreVO, (quotaDelta * Resource.ResourceType.bytesToGiB)); } _bucketDao.update(bucket.getId(), bucket); } catch (Exception e) { @@ -313,9 +328,11 @@ protected void runInContext() { continue; } List buckets = _bucketDao.listByObjectStoreId(objectStoreVO.getId()); + Long objectStoreUsedBytes = 0L; for(BucketVO bucket : buckets) { Long size = bucketSizes.get(bucket.getName()); - if( size != null){ + if( size != null) { + objectStoreUsedBytes += size; bucket.setSize(size); _bucketDao.update(bucket.getId(), bucket); @@ -331,6 +348,8 @@ protected void runInContext() { } } } + objectStoreVO.setUsedSize(objectStoreUsedBytes); + _objectStoreDao.persist(objectStoreVO); } logger.debug("Completed updating bucket usage for all object stores"); } catch (Exception e) { diff --git a/server/src/test/java/com/cloud/alert/AlertManagerImplTest.java b/server/src/test/java/com/cloud/alert/AlertManagerImplTest.java index d34d0b5873f2..170fceae9861 100644 --- a/server/src/test/java/com/cloud/alert/AlertManagerImplTest.java +++ b/server/src/test/java/com/cloud/alert/AlertManagerImplTest.java @@ -17,10 +17,15 @@ package com.cloud.alert; import java.io.UnsupportedEncodingException; +import java.util.HashMap; import java.util.List; +import java.util.Optional; import javax.mail.MessagingException; +import javax.naming.ConfigurationException; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.mailing.SMTPMailSender; @@ -39,6 +44,8 @@ import com.cloud.alert.dao.AlertDao; import com.cloud.capacity.Capacity; import com.cloud.capacity.CapacityManager; +import com.cloud.capacity.CapacityVO; +import com.cloud.capacity.dao.CapacityDao; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; @@ -96,6 +103,15 @@ public class AlertManagerImplTest { @Mock SMTPMailSender mailSenderMock; + @Mock + CapacityDao capacityDao; + + @Mock + BackupManager backupManager; + + @Mock + ConfigurationDao configDao; + private final String[] recipients = new String[]{"test@test.com"}; private final String senderAddress = "sender@test.com"; @@ -219,4 +235,37 @@ public void testRecalculateStorageCapacities() { Mockito.verify(storageManager, Mockito.times(2)).createCapacityEntry(sharedPool, Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED, 10L); Mockito.verify(storageManager, Mockito.times(1)).createCapacityEntry(nonSharedPool, Capacity.CAPACITY_TYPE_LOCAL_STORAGE, 20L); } + + @Test + public void testCheckForAlerts() throws ConfigurationException { + Long zoneId = 1L; + Mockito.doNothing().when(alertManagerImplMock).recalculateCapacity(); + DataCenterVO dc = Mockito.mock(DataCenterVO.class); + Mockito.when(dc.getId()).thenReturn(zoneId); + Mockito.when(dc.getName()).thenReturn("zone1"); + Mockito.when(_dcDao.listAll()).thenReturn(List.of(dc)); + Mockito.when(_dcDao.findById(zoneId)).thenReturn(dc); + Mockito.when(configDao.getConfiguration("management-server", null)).thenReturn(new HashMap<>()); + + alertManagerImplMock.configure(null, null); + CapacityVO secondaryStorageCapacity = new CapacityVO(null, zoneId, null, null, 100L, 200L, Capacity.CAPACITY_TYPE_SECONDARY_STORAGE); + CapacityVO storagePoolCapacity = new CapacityVO(null, zoneId, null, null, 200L, 300L, Capacity.CAPACITY_TYPE_STORAGE); + CapacityVO objectStoreCapacity = new CapacityVO(null, zoneId, null, null, 200L, 300L, Capacity.CAPACITY_TYPE_OBJECT_STORAGE); + CapacityVO backupCapacity = new CapacityVO(null, zoneId, null, null, 180L, 200L, Capacity.CAPACITY_TYPE_BACKUP_STORAGE); + Mockito.when(storageManager.getSecondaryStorageUsedStats(null, zoneId)).thenReturn(secondaryStorageCapacity); + Mockito.when(storageManager.getObjectStorageUsedStats(zoneId)).thenReturn(objectStoreCapacity); + Mockito.when(backupManager.getBackupStorageUsedStats(zoneId)).thenReturn(backupCapacity); + alertManagerImplMock.checkForAlerts(); + + Mockito.verify(alertManagerImplMock).recalculateCapacity(); + + ArgumentCaptor alertCaptor = ArgumentCaptor.forClass(AlertVO.class); + verify(_alertDao).persist(alertCaptor.capture()); + AlertVO capturedAlert = alertCaptor.getValue(); + assertNotNull("Captured alert should not be null", capturedAlert); + assertEquals(Optional.of(zoneId), Optional.ofNullable(capturedAlert.getDataCenterId())); + assertEquals("System Alert: Low Available Backup Storage in availability zone zone1", capturedAlert.getSubject()); + assertEquals("Available backup storage space is low, total: 200.0 MB, used: 180.0 MB (90%)", capturedAlert.getContent()); + assertEquals(AlertManager.AlertType.ALERT_TYPE_BACKUP_STORAGE.getType(), capturedAlert.getType()); + } } diff --git a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java index 30350d71684e..2189b4517614 100644 --- a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java +++ b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java @@ -1268,4 +1268,35 @@ public void testGetNicVlanValueForExternalVm_Nsx_NoBean() { service.getNicVlanValueForExternalVm(nic); } } + + @Test + public void testGetIpAddressesFromIps() { + // Test with valid IPv4, IPv6 and MAC address + Network.IpAddresses result = service.getIpAddressesFromIps("192.168.1.1", "2001:db8::1", "00:11:22:33:44:55"); + Assert.assertEquals("192.168.1.1", result.getIp4Address()); + Assert.assertEquals("2001:db8::1", result.getIp6Address()); + Assert.assertEquals("00:11:22:33:44:55", result.getMacAddress()); + + // Test with all null values + result = service.getIpAddressesFromIps(null, null, null); + Assert.assertNull(result.getIp4Address()); + Assert.assertNull(result.getIp6Address()); + Assert.assertNull(result.getMacAddress()); + + // Test with invalid MAC address (non-unicast) + try { + service.getIpAddressesFromIps(null, null, "ff:ff:ff:ff:ff:ff"); + Assert.fail("Expected InvalidParameterValueException for non-unicast MAC address"); + } catch (InvalidParameterValueException e) { + Assert.assertEquals("Mac address is not unicast: ff:ff:ff:ff:ff:ff", e.getMessage()); + } + + // Test with invalid MAC address (invalid format) + try { + service.getIpAddressesFromIps(null, null, "invalid-mac"); + Assert.fail("Expected InvalidParameterValueException for invalid MAC address format"); + } catch (InvalidParameterValueException e) { + Assert.assertEquals("Mac address is not valid: invalid-mac", e.getMessage()); + } + } } diff --git a/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java b/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java index 510cd7ac1a6e..7ddd0d612138 100644 --- a/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java +++ b/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java @@ -1272,7 +1272,7 @@ public void testCreateNewVM1() throws ResourceUnavailableException, Insufficient when(zoneMock.getNetworkType()).thenReturn(DataCenter.NetworkType.Basic); when(userVmService.createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(), - any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), eq(true), any(), any(), any(), + any(), any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), eq(true), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any())).thenReturn(userVmMock); UserVm result = autoScaleManagerImplSpy.createNewVM(asVmGroupMock); @@ -1283,7 +1283,7 @@ public void testCreateNewVM1() throws ResourceUnavailableException, Insufficient "-" + asVmGroupMock.getNextVmSeq() + "-[a-z]{6}"; Mockito.verify(userVmService).createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), matches(vmHostNamePattern), matches(vmHostNamePattern), - any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), eq(true), any(), any(), any(), + any(), any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), eq(true), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any()); Mockito.verify(asVmGroupMock).setNextVmSeq(nextVmSeq + 1); } @@ -1319,7 +1319,7 @@ public void testCreateNewVM2() throws ResourceUnavailableException, Insufficient when(zoneMock.getNetworkType()).thenReturn(DataCenter.NetworkType.Advanced); when(userVmService.createAdvancedSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(), - any(), any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any(), any())).thenReturn(userVmMock); when(networkModel.checkSecurityGroupSupportForNetwork(account, zoneMock, List.of(networkId), Collections.emptyList())).thenReturn(true); @@ -1332,7 +1332,7 @@ public void testCreateNewVM2() throws ResourceUnavailableException, Insufficient "-" + asVmGroupMock.getNextVmSeq() + "-[a-z]{6}"; Mockito.verify(userVmService).createAdvancedSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), matches(vmHostNamePattern), matches(vmHostNamePattern), - any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any(), any()); Mockito.verify(asVmGroupMock).setNextVmSeq(nextVmSeq + 2); } @@ -1368,7 +1368,7 @@ public void testCreateNewVM3() throws ResourceUnavailableException, Insufficient when(zoneMock.getNetworkType()).thenReturn(DataCenter.NetworkType.Advanced); when(userVmService.createAdvancedVirtualMachine(any(), any(), any(), any(), any(), any(), any(), - any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), eq(true), any(), any(), any(), + any(), any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), eq(true), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any(), any())).thenReturn(userVmMock); when(networkModel.checkSecurityGroupSupportForNetwork(account, zoneMock, List.of(networkId), Collections.emptyList())).thenReturn(false); @@ -1381,7 +1381,7 @@ public void testCreateNewVM3() throws ResourceUnavailableException, Insufficient "-" + asVmGroupMock.getNextVmSeq() + "-[a-z]{6}"; Mockito.verify(userVmService).createAdvancedVirtualMachine(any(), any(), any(), any(), any(), matches(vmHostNamePattern), matches(vmHostNamePattern), - any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), eq(true), any(), any(), any(), + any(), any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), eq(true), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any(), any()); Mockito.verify(asVmGroupMock).setNextVmSeq(nextVmSeq + 3); } diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java index 01ab82c913dd..6eb8bd04f46d 100644 --- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import com.cloud.dc.HostPodVO; import com.cloud.dc.dao.HostPodDao; @@ -32,16 +33,24 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.storage.ChangeStoragePoolScopeCmd; import org.apache.cloudstack.api.command.admin.storage.ConfigureStorageAccessCmd; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.object.ObjectStore; import org.apache.commons.collections.MapUtils; import org.junit.Assert; import org.junit.Test; @@ -57,7 +66,9 @@ import com.cloud.agent.AgentManager; import com.cloud.agent.api.Command; import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.capacity.Capacity; import com.cloud.capacity.CapacityManager; +import com.cloud.capacity.CapacityVO; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; @@ -150,6 +161,15 @@ public class StorageManagerImplTest { private ResourceManager resourceMgr; + @Mock + protected ObjectStoreDao objectStoreDao; + + @Mock + DataStoreProviderManager dataStoreProviderMgr; + + @Mock + DataStoreManager dataStoreMgr; + @Test public void createLocalStoragePoolName() { String hostMockName = "host1"; @@ -255,7 +275,7 @@ public void storagePoolCompatibleWithVolumePoolTestVolumeWithoutPoolIdInAllocate volume.setState(Volume.State.Allocated); PrimaryDataStoreDao storagePoolDao = Mockito.mock(PrimaryDataStoreDao.class); storageManagerImpl._storagePoolDao = storagePoolDao; - assertTrue(storageManagerImpl.storagePoolCompatibleWithVolumePool(storagePool, volume)); + Assert.assertTrue(storageManagerImpl.storagePoolCompatibleWithVolumePool(storagePool, volume)); } @@ -264,8 +284,8 @@ public void testExtractUriParamsAsMapWithSolidFireUrl() { String sfUrl = "MVIP=1.2.3.4;SVIP=6.7.8.9;clusterAdminUsername=admin;" + "clusterAdminPassword=password;clusterDefaultMinIops=1000;" + "clusterDefaultMaxIops=2000;clusterDefaultBurstIopsPercentOfMaxIops=2"; - Map uriParams = storageManagerImpl.extractUriParamsAsMap(sfUrl); - assertTrue(MapUtils.isEmpty(uriParams)); + Map uriParams = storageManagerImpl.extractUriParamsAsMap(sfUrl); + Assert.assertTrue(MapUtils.isEmpty(uriParams)); } @Test @@ -274,8 +294,8 @@ public void testExtractUriParamsAsMapWithNFSUrl() { String host = "HOST"; String path = "/PATH"; String sfUrl = String.format("%s://%s%s", scheme, host, path); - Map uriParams = storageManagerImpl.extractUriParamsAsMap(sfUrl); - assertTrue(MapUtils.isNotEmpty(uriParams)); + Map uriParams = storageManagerImpl.extractUriParamsAsMap(sfUrl); + Assert.assertTrue(MapUtils.isNotEmpty(uriParams)); Assert.assertEquals(scheme, uriParams.get("scheme")); Assert.assertEquals(host, uriParams.get("host")); Assert.assertEquals(path, uriParams.get("hostPath")); @@ -433,7 +453,7 @@ public void testIsStoragePoolCompliantWithStoragePolicy() { } try { Mockito.doReturn(new com.cloud.agent.api.Answer( - Mockito.mock(CheckDataStoreStoragePolicyComplainceCommand.class))) + Mockito.mock(CheckDataStoreStoragePolicyComplainceCommand.class))) .when(storageManagerImpl).getCheckDatastorePolicyComplianceAnswer("policy", pool); assertTrue(storageManagerImpl.isStoragePoolCompliantWithStoragePolicy(1L, pool)); } catch (StorageUnavailableException e) { @@ -480,8 +500,7 @@ public void testGetCheckDatastorePolicyComplianceAnswerAgentException() throws S Mockito.when(policy.getPolicyId()).thenReturn("some"); Mockito.when(vsphereStoragePolicyDao.findById(Mockito.anyLong())) .thenReturn(policy); - Mockito.doReturn(new ArrayList<>(List.of(1L, 2L))) - .when(storageManagerImpl).getUpHostsInPool(Mockito.anyLong()); + Mockito.doReturn(new ArrayList<>(List.of(1L, 2L))).when(storageManagerImpl).getUpHostsInPool(Mockito.anyLong()); Mockito.when(hvGuruMgr.getGuruProcessedCommandTargetHost(Mockito.anyLong(), Mockito.any(CheckDataStoreStoragePolicyComplainceCommand.class))).thenReturn(1L); try { @@ -552,7 +571,7 @@ public void testEnableDefaultDatastoreDownloadRedirectionForExistingInstallation .thenReturn(new ArrayList<>()); //new installation storageManagerImpl.enableDefaultDatastoreDownloadRedirectionForExistingInstallations(); Mockito.verify(configurationDao, Mockito.never()) - .update(StorageManager.DataStoreDownloadFollowRedirects.key(),StorageManager.DataStoreDownloadFollowRedirects.defaultValue()); + .update(StorageManager.DataStoreDownloadFollowRedirects.key(), StorageManager.DataStoreDownloadFollowRedirects.defaultValue()); } @Test @@ -796,7 +815,7 @@ private Long testCheckPoolforSpaceForResizeSetup(StoragePoolVO pool, Long alloca Long zoneId = 2L; Long capacityBytes = (long) (allocatedSizeWithTemplate / Double.valueOf(CapacityManager.StorageAllocatedCapacityDisableThreshold.defaultValue()) - / Double.valueOf(CapacityManager.StorageOverprovisioningFactor.defaultValue())); + / Double.valueOf(CapacityManager.StorageOverprovisioningFactor.defaultValue())); Long maxAllocatedSizeForResize = (long) (capacityBytes * Double.valueOf(CapacityManager.StorageOverprovisioningFactor.defaultValue()) * Double.valueOf(CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.defaultValue())); @@ -1487,4 +1506,162 @@ public void testCheckIfStorageAccessGroupsExistsOnCluster_ThrowsException() { assertTrue(thrownException.getMessage().contains("access groups already exist on the cluster: [group4]")); } + + @Test + public void testGetObjectStorageUsedStats() { + Long zoneId = 1L; + List objectStores = new ArrayList<>(); + + ObjectStoreVO store1 = new ObjectStoreVO(); + store1.setAllocatedSize(1000L); + store1.setTotalSize(2000L); + objectStores.add(store1); + + ObjectStoreVO store2 = new ObjectStoreVO(); + store2.setAllocatedSize(2000L); + store2.setTotalSize(4000L); + objectStores.add(store2); + + ObjectStoreVO store3 = new ObjectStoreVO(); + store3.setAllocatedSize(null); + store3.setTotalSize(null); + objectStores.add(store3); + + Mockito.when(objectStoreDao.listObjectStores()).thenReturn(objectStores); + + CapacityVO result = storageManagerImpl.getObjectStorageUsedStats(zoneId); + + Assert.assertEquals(zoneId, result.getDataCenterId()); + Assert.assertEquals(Optional.of(3000L), Optional.of(result.getUsedCapacity())); // 1000 + 2000 + Assert.assertEquals(6000L, result.getTotalCapacity()); // 2000 + 4000 + Assert.assertEquals(Capacity.CAPACITY_TYPE_OBJECT_STORAGE, result.getCapacityType()); + Assert.assertNull(result.getPodId()); + Assert.assertNull(result.getClusterId()); + } + + @Test + public void testGetObjectStorageUsedStatsWithNullSizes() { + Long zoneId = 1L; + List objectStores = new ArrayList<>(); + + ObjectStoreVO store1 = new ObjectStoreVO(); + store1.setAllocatedSize(null); + store1.setTotalSize(null); + objectStores.add(store1); + + ObjectStoreVO store2 = new ObjectStoreVO(); + store2.setAllocatedSize(null); + store2.setTotalSize(null); + objectStores.add(store2); + + Mockito.when(objectStoreDao.listObjectStores()).thenReturn(objectStores); + + CapacityVO result = storageManagerImpl.getObjectStorageUsedStats(zoneId); + + Assert.assertEquals(zoneId, result.getDataCenterId()); + Assert.assertEquals(Optional.of(0L), Optional.of(result.getUsedCapacity())); + Assert.assertEquals(0L, result.getTotalCapacity()); + Assert.assertEquals(Capacity.CAPACITY_TYPE_OBJECT_STORAGE, result.getCapacityType()); + Assert.assertNull(result.getPodId()); + Assert.assertNull(result.getClusterId()); + } + + @Test + public void testDiscoverObjectStore() { + Long objectStoreId = 1L; + + String name = "test-store"; + String url = "http://10.1.1.33:80"; + Long size = 1000L; + String providerName = "test-provider"; + Map details = new HashMap<>(); + details.put("key1", "value1"); + + ObjectStoreVO objectStoreVO = new ObjectStoreVO(); + ReflectionTestUtils.setField(objectStoreVO, "id", objectStoreId); + objectStoreVO.setName(name); + objectStoreVO.setUrl(url); + objectStoreVO.setProviderName(providerName); + objectStoreVO.setTotalSize(size); + + DataStoreProvider storeProvider = Mockito.mock(DataStoreProvider.class); + DataStoreLifeCycle lifeCycle = Mockito.mock(DataStoreLifeCycle.class); + DataStore store = Mockito.mock(DataStore.class); + ObjectStore objectStore = Mockito.mock(ObjectStore.class); + + Mockito.when(dataStoreProviderMgr.getDataStoreProvider(providerName)).thenReturn(storeProvider); + Mockito.when(storeProvider.getDataStoreLifeCycle()).thenReturn(lifeCycle); + Mockito.when(lifeCycle.initialize(Mockito.any())).thenReturn(store); + Mockito.when(store.getId()).thenReturn(1L); + Mockito.when(dataStoreMgr.getDataStore(1L, DataStoreRole.Object)).thenReturn(null); + + ObjectStore result = storageManagerImpl.discoverObjectStore(name, url, size, providerName, details); + + Mockito.verify(dataStoreProviderMgr).getDataStoreProvider(providerName); + Mockito.verify(lifeCycle).initialize(Mockito.any()); + Mockito.verify(dataStoreMgr).getDataStore(1L, DataStoreRole.Object); + } + + @Test(expected = InvalidParameterValueException.class) + public void testDiscoverObjectStoreInvalidProvider() { + // Setup + String name = "test-store"; + String url = "http://10.1.1.33:80"; + Long size = 1000L; + String providerName = "invalid-provider"; + Map details = new HashMap<>(); + + Mockito.when(dataStoreProviderMgr.getDataStoreProvider(providerName)).thenReturn(null); + + storageManagerImpl.discoverObjectStore(name, url, size, providerName, details); + } + + @Test(expected = IllegalArgumentException.class) + public void testDiscoverObjectStoreInvalidUrl() { + String name = "test-store"; + String url = "invalid-url"; + Long size = 1000L; + String providerName = "test-provider"; + Map details = new HashMap<>(); + + DataStoreProvider storeProvider = Mockito.mock(DataStoreProvider.class); + Mockito.when(dataStoreProviderMgr.getDataStoreProvider(providerName)).thenReturn(storeProvider); + + storageManagerImpl.discoverObjectStore(name, url, size, providerName, details); + } + + @Test(expected = InvalidParameterValueException.class) + public void testDiscoverObjectStoreDuplicateUrl() { + String name = "test-store"; + String url = "http://10.1.1.33:80"; + Long size = 1000L; + String providerName = "test-provider"; + Map details = new HashMap<>(); + + DataStoreProvider storeProvider = Mockito.mock(DataStoreProvider.class); + ObjectStoreVO existingStore = new ObjectStoreVO(); + + Mockito.when(dataStoreProviderMgr.getDataStoreProvider(providerName)).thenReturn(storeProvider); + Mockito.when(objectStoreDao.findByUrl(url)).thenReturn(existingStore); + + storageManagerImpl.discoverObjectStore(name, url, size, providerName, details); + } + + @Test(expected = CloudRuntimeException.class) + public void testDiscoverObjectStoreInitializationFailure() { + String name = "test-store"; + String url = "http://10.1.1.33:80"; + Long size = 1000L; + String providerName = "test-provider"; + Map details = new HashMap<>(); + + DataStoreProvider storeProvider = Mockito.mock(DataStoreProvider.class); + DataStoreLifeCycle lifeCycle = Mockito.mock(DataStoreLifeCycle.class); + + Mockito.when(dataStoreProviderMgr.getDataStoreProvider(providerName)).thenReturn(storeProvider); + Mockito.when(storeProvider.getDataStoreLifeCycle()).thenReturn(lifeCycle); + Mockito.when(lifeCycle.initialize(Mockito.any())).thenThrow(new RuntimeException("Initialization failed")); + + storageManagerImpl.discoverObjectStore(name, url, size, providerName, details); + } } diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index b25da259b6bb..e1efd87dcd8b 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -29,6 +29,7 @@ import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -43,6 +44,7 @@ import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -56,14 +58,24 @@ import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.BaseCmd.HTTPMethod; import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd; +import org.apache.cloudstack.api.command.admin.vm.ExpungeVMCmd; +import org.apache.cloudstack.api.command.user.vm.CreateVMFromBackupCmd; import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; import org.apache.cloudstack.api.command.user.vm.DeployVnfApplianceCmd; +import org.apache.cloudstack.api.command.user.vm.DestroyVMCmd; +import org.apache.cloudstack.api.command.user.vm.ResetVMSSHKeyCmd; import org.apache.cloudstack.api.command.user.vm.ResetVMUserDataCmd; import org.apache.cloudstack.api.command.user.vm.RestoreVMCmd; import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.backup.BackupVO; +import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; @@ -115,6 +127,7 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.network.Network; import com.cloud.network.NetworkModel; +import com.cloud.network.as.AutoScaleManager; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; @@ -122,6 +135,7 @@ import com.cloud.network.dao.LoadBalancerVMMapVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.element.UserDataServiceProvider; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkVO; import com.cloud.network.guru.NetworkGuru; @@ -159,15 +173,18 @@ import com.cloud.user.AccountService; import com.cloud.user.AccountVO; import com.cloud.user.ResourceLimitService; +import com.cloud.user.SSHKeyPairVO; import com.cloud.user.UserData; import com.cloud.user.UserDataVO; import com.cloud.user.UserVO; import com.cloud.user.dao.AccountDao; +import com.cloud.user.dao.SSHKeyPairDao; import com.cloud.user.dao.UserDao; import com.cloud.user.dao.UserDataDao; import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; import com.cloud.utils.db.EntityManager; +import com.cloud.utils.db.UUIDManager; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.ExceptionProxyObject; import com.cloud.vm.dao.NicDao; @@ -264,6 +281,12 @@ public class UserVmManagerImplTest { @Mock PrimaryDataStoreDao primaryDataStoreDao; + @Mock + BackupDao backupDao; + + @Mock + BackupManager backupManager; + @Mock VirtualMachineManager virtualMachineManager; @@ -378,6 +401,9 @@ public class UserVmManagerImplTest { @Mock ServiceOfferingJoinDao serviceOfferingJoinDao; + @Mock + SSHKeyPairDao sshKeyPairDao; + @Mock private VMInstanceVO vmInstanceMock; @@ -399,6 +425,12 @@ public class UserVmManagerImplTest { @Mock private Scope scopeMock; + @Mock + private AutoScaleManager autoScaleManager; + + @Mock + private UUIDManager uuidMgr; + private static final long vmId = 1l; private static final long zoneId = 2L; private static final long accountId = 3L; @@ -634,13 +666,13 @@ private void configureDoNothingForMethodsThatWeDoNotWantToTest() throws Resource Mockito.lenient().doReturn(Mockito.mock(UserVm.class)).when(userVmManagerImpl).updateVirtualMachine(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyBoolean(), Mockito.any(HTTPMethod.class), Mockito.anyString(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyList(), Mockito.any()); + Mockito.anyString(), anyList(), Mockito.any()); Mockito.doNothing().when(userVmManagerImpl).validateIfVmSupportsMigration(Mockito.any(), Mockito.anyLong()); Mockito.doNothing().when(userVmManagerImpl).validateOldAndNewAccounts(Mockito.nullable(Account.class), Mockito.nullable(Account.class), Mockito.anyLong(), Mockito.nullable(String.class), Mockito.nullable(Long.class)); Mockito.doNothing().when(userVmManagerImpl).validateIfVmHasNoRules(Mockito.any(), Mockito.anyLong()); Mockito.doNothing().when(userVmManagerImpl).removeInstanceFromInstanceGroup(Mockito.anyLong()); - Mockito.doNothing().when(userVmManagerImpl).verifyResourceLimitsForAccountAndStorage(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyList(), Mockito.any()); + Mockito.doNothing().when(userVmManagerImpl).verifyResourceLimitsForAccountAndStorage(Mockito.any(), Mockito.any(), Mockito.any(), anyList(), Mockito.any()); Mockito.doNothing().when(userVmManagerImpl).validateIfNewOwnerHasAccessToTemplate(Mockito.any(), Mockito.any(), Mockito.any()); Mockito.doNothing().when(userVmManagerImpl).updateVmOwner(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); @@ -1118,14 +1150,14 @@ public void createVirtualMachine() throws ResourceUnavailableException, Insuffic when(_dcMock.isLocalStorageEnabled()).thenReturn(true); when(_dcMock.getNetworkType()).thenReturn(DataCenter.NetworkType.Basic); Mockito.doReturn(userVmVoMock).when(userVmManagerImpl).createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(), - any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any()); UserVm result = userVmManagerImpl.createVirtualMachine(deployVMCmd); assertEquals(userVmVoMock, result); Mockito.verify(vnfTemplateManager).validateVnfApplianceNics(templateMock, null); Mockito.verify(userVmManagerImpl).createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(), - any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any()); } @@ -1378,7 +1410,7 @@ public void createVirtualMachineWithCloudRuntimeException() throws ResourceUnava cre.addProxyObject(vmId, "vmId"); Mockito.doThrow(cre).when(userVmManagerImpl).createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(), - any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any()); CloudRuntimeException creThrown = assertThrows(CloudRuntimeException.class, () -> userVmManagerImpl.createVirtualMachine(deployVMCmd)); @@ -2718,7 +2750,7 @@ public void updateBasicTypeNetworkForVmTestNetworkIdListIsNullCallsCleanupOfOldO securityGroupIdList); Mockito.verify(userVmManagerImpl).cleanupOfOldOwnerNicsForNetwork(virtualMachineProfileMock); - Mockito.verify(userVmManagerImpl).addDefaultNetworkToNetworkList(Mockito.anyList(), Mockito.any()); + Mockito.verify(userVmManagerImpl).addDefaultNetworkToNetworkList(anyList(), Mockito.any()); Mockito.verify(userVmManagerImpl).allocateNetworksForVm(Mockito.any(), Mockito.any()); Mockito.verify(userVmManagerImpl).addSecurityGroupsToVm(accountMock, userVmVoMock,virtualMachineTemplateMock, securityGroupIdList, networkMock); } @@ -2736,7 +2768,7 @@ public void updateBasicTypeNetworkForVmTestNetworkIdListIsEmptyCallsCleanupOfOld securityGroupIdList); Mockito.verify(userVmManagerImpl).cleanupOfOldOwnerNicsForNetwork(virtualMachineProfileMock); - Mockito.verify(userVmManagerImpl).addDefaultNetworkToNetworkList(Mockito.anyList(), Mockito.any()); + Mockito.verify(userVmManagerImpl).addDefaultNetworkToNetworkList(anyList(), Mockito.any()); Mockito.verify(userVmManagerImpl).allocateNetworksForVm(Mockito.any(), Mockito.any()); Mockito.verify(userVmManagerImpl).addSecurityGroupsToVm(accountMock, userVmVoMock,virtualMachineTemplateMock, securityGroupIdList, networkMock); } @@ -3243,6 +3275,392 @@ public void validateNullStorageAccessGroupsOnSrcHost() { Mockito.verify(storageManager, times(1)).getStorageAccessGroups(null, null, null, srcHost.getId()); Mockito.verify(storageManager, times(1)).getStorageAccessGroups(null, null, null, destHost.getId()); } + @Test + public void testAllocateVMFromBackupUsingCmdValues() throws InsufficientCapacityException, ResourceAllocationException, ResourceUnavailableException { + Long backupId = 4L; + + CreateVMFromBackupCmd cmd = new CreateVMFromBackupCmd(); + cmd._accountService = accountService; + cmd._entityMgr = entityManager; + when(accountService.finalyzeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); + when(accountService.getActiveAccountById(accountId)).thenReturn(account); + + ReflectionTestUtils.setField(cmd, "serviceOfferingId", serviceOfferingId); + ReflectionTestUtils.setField(cmd, "templateId", templateId); + ReflectionTestUtils.setField(cmd, "backupId", backupId); + ReflectionTestUtils.setField(cmd, "zoneId", zoneId); + + ServiceOfferingVO serviceOffering = mock(ServiceOfferingVO.class); + when(_serviceOfferingDao.findById(serviceOfferingId)).thenReturn(serviceOffering); + + Map diskDetails = new HashMap<>(); + diskDetails.put(ApiConstants.DISK_OFFERING_ID, "disk-offering-uuid"); + diskDetails.put(ApiConstants.DEVICE_ID, "1"); + diskDetails.put(ApiConstants.SIZE, "5"); + diskDetails.put(ApiConstants.MIN_IOPS, "1000"); + diskDetails.put(ApiConstants.MAX_IOPS, "5000"); + Map> disksDetails = new HashMap<>(); + disksDetails.put(0, diskDetails); + ReflectionTestUtils.setField(cmd, "dataDisksDetails", disksDetails); + DiskOffering diskOffering = mock(DiskOffering.class); + when(diskOffering.isCustomized()).thenReturn(true); + when(diskOffering.isCustomizedIops()).thenReturn(true); + when(entityManager.findByUuid(DiskOffering.class, "disk-offering-uuid")).thenReturn(diskOffering); + + BackupVO backup = mock(BackupVO.class); + when(backup.getZoneId()).thenReturn(zoneId); + when(backup.getVmId()).thenReturn(vmId); + when(backupDao.findById(backupId)).thenReturn(backup); + + UserVmVO userVmVO = new UserVmVO(); + userVmVO.setTemplateId(templateId); + when(userVmDao.findByIdIncludingRemoved(vmId)).thenReturn(userVmVO); + VMTemplateVO template = mock(VMTemplateVO.class); + when(template.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); + when(templateDao.findById(templateId)).thenReturn(template); + VmDiskInfo rootVmDiskInfo = new VmDiskInfo(diskOffering, 10L, 1000L, 2000L); + when(backupManager.getRootDiskInfoFromBackup(backup)).thenReturn(rootVmDiskInfo); + Mockito.when(backupManager.canCreateInstanceFromBackup(backupId)).thenReturn(true); + + Mockito.doReturn(userVmVoMock).when(userVmManagerImpl).createAdvancedVirtualMachine(any(), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), + any(), any(), any(), any(), eq(true), any(), any(), any(), any()); + + UserVm result = userVmManagerImpl.allocateVMFromBackup(cmd); + + assertNotNull(result); + Mockito.verify(backupDao).findById(backupId); + Mockito.verify(userVmManagerImpl).createAdvancedVirtualMachine(any(), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), + any(), any(), any(), any(), eq(true), any(), any(), any(), any()); + } + + @Test + public void testAllocateVMFromBackupUsingBackupValues() throws InsufficientCapacityException, ResourceAllocationException, ResourceUnavailableException { + Long backupId = 5L; + + CreateVMFromBackupCmd cmd = mock(CreateVMFromBackupCmd.class); + when(cmd.getZoneId()).thenReturn(zoneId); + when(cmd.getBackupId()).thenReturn(backupId); + when(cmd.getEntityOwnerId()).thenReturn(accountId); + when(cmd.getServiceOfferingId()).thenReturn(null); + when(cmd.getHostId()).thenReturn(null); + when(cmd.getDiskOfferingId()).thenReturn(null); + when(cmd.getTemplateId()).thenReturn(null); + when(cmd.getNetworkIds()).thenReturn(null); + when(cmd.getIpToNetworkMap()).thenReturn(null); + when(cmd.getDataDiskInfoList()).thenReturn(null); + when(cmd.getOverrideDiskOfferingId()).thenReturn(null); + + Account owner = mock(Account.class); + when(accountService.getActiveAccountById(accountId)).thenReturn(owner); + + DataCenterVO zone = mock(DataCenterVO.class); + when(_dcDao.findById(zoneId)).thenReturn(zone); + + BackupVO backup = mock(BackupVO.class); + when(backup.getZoneId()).thenReturn(zoneId); + when(backup.getVmId()).thenReturn(vmId); + when(backupDao.findById(backupId)).thenReturn(backup); + + UserVmVO userVmVO = new UserVmVO(); + when(userVmDao.findByIdIncludingRemoved(vmId)).thenReturn(userVmVO); + VMTemplateVO template = mock(VMTemplateVO.class); + when(template.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); + when(backup.getDetail(ApiConstants.TEMPLATE_ID)).thenReturn("template-uuid"); + when(templateDao.findByUuid("template-uuid")).thenReturn(template); + + DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); + when(backup.getDetail(ApiConstants.SERVICE_OFFERING_ID)).thenReturn("service-offering-uuid"); + when(_serviceOfferingDao.findByUuid("service-offering-uuid")).thenReturn(serviceOffering); + VmDiskInfo rootVmDiskInfo = new VmDiskInfo(diskOffering, 10L, 1000L, 2000L); + when(backupManager.getRootDiskInfoFromBackup(backup)).thenReturn(rootVmDiskInfo); + + NetworkVO network1 = mock(NetworkVO.class); + NetworkVO network2 = mock(NetworkVO.class); + when(backupManager.getDataDiskInfoListFromBackup(backup)).thenReturn(List.of(new VmDiskInfo(diskOffering, 10L, 1000L, 2000L))); + Mockito.when(backupManager.canCreateInstanceFromBackup(backupId)).thenReturn(true); + + Mockito.doReturn(userVmVoMock).when(userVmManagerImpl).createAdvancedVirtualMachine(any(), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), eq(false), any(), any(), any(), + any(), any(), any(), any(), eq(false), any(), any(), any(), any()); + + UserVm result = userVmManagerImpl.allocateVMFromBackup(cmd); + + assertNotNull(result); + Mockito.verify(userVmManagerImpl).createAdvancedVirtualMachine(any(), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), eq(false), any(), any(), any(), + any(), any(), any(), any(), eq(false), any(), any(), any(), any()); + } + + @Test + public void testResetVMSSHKey() throws ResourceUnavailableException, InsufficientCapacityException { + Long domainId = 4L; + Long projectId = 5L; + Long networkId = 6L; + + List names = List.of("keypair1", "keypair2"); + ResetVMSSHKeyCmd cmd = mock(ResetVMSSHKeyCmd.class); + when(cmd.getId()).thenReturn(vmId); + when(cmd.getAccountName()).thenReturn("testAccount"); + when(cmd.getDomainId()).thenReturn(domainId); + when(cmd.getProjectId()).thenReturn(projectId); + when(cmd.getNames()).thenReturn(names); + + Account owner = mock(Account.class); + when(owner.getAccountId()).thenReturn(accountId); + when(owner.getDomainId()).thenReturn(domainId); + when(accountManager.finalizeOwner(callerAccount, "testAccount", domainId, projectId)).thenReturn(owner); + + UserVmVO userVm = new UserVmVO(vmId, null, null, templateId, Hypervisor.HypervisorType.KVM, 0, + true, false, domainId, accountId, 0L, 0L, null, null, null, null); + ReflectionTestUtils.setField(userVm, "state", VirtualMachine.State.Stopped); + userVm.setUserVmType("User"); + when(userVmDao.findById(vmId)).thenReturn(userVm); + VMTemplateVO template = mock(VMTemplateVO.class); + when(templateDao.findByIdIncludingRemoved(templateId)).thenReturn(template); + + Nic nic = mock(Nic.class); + when(nic.getNetworkId()).thenReturn(networkId); + when(networkModel.getDefaultNic(vmId)).thenReturn(nic); + NetworkVO network = mock(NetworkVO.class); + when(_networkDao.findById(networkId)).thenReturn(network); + UserDataServiceProvider element = mock(UserDataServiceProvider.class); + when(element.saveSSHKey(any(), any(), any(), any())).thenReturn(true); + when(_networkMgr.getSSHKeyResetProvider(network)).thenReturn(element); + + SSHKeyPairVO keyPair1 = mock(SSHKeyPairVO.class); + SSHKeyPairVO keyPair2 = mock(SSHKeyPairVO.class); + when(keyPair1.getPublicKey()).thenReturn("ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAr..."); + when(keyPair2.getPublicKey()).thenReturn("ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAr..."); + when(sshKeyPairDao.findByNames(accountId, domainId, names)).thenReturn(Arrays.asList(keyPair1, keyPair2)); + + UserVm result = userVmManagerImpl.resetVMSSHKey(cmd); + + assertNotNull(result); + Map details = result.getDetails(); + Assert.assertEquals(details.get(VmDetailConstants.SSH_PUBLIC_KEY), "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAr...\n" + + "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAr..."); + Assert.assertEquals(details.get(VmDetailConstants.SSH_KEY_PAIR_NAMES), "keypair1,keypair2"); + } + + @Test + public void testAllocateVMFromBackupUsingCmdValuesWithISO() throws InsufficientCapacityException, ResourceAllocationException, ResourceUnavailableException { + Long backupId = 4L; + Long rootDiskOfferingId = 5L; + Long isoId = 7L; + + CreateVMFromBackupCmd cmd = new CreateVMFromBackupCmd(); + cmd._accountService = accountService; + cmd._entityMgr = entityManager; + when(accountService.finalyzeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); + when(accountService.getActiveAccountById(accountId)).thenReturn(account); + + ReflectionTestUtils.setField(cmd, "serviceOfferingId", serviceOfferingId); + ReflectionTestUtils.setField(cmd, "templateId", isoId); + ReflectionTestUtils.setField(cmd, "backupId", backupId); + ReflectionTestUtils.setField(cmd, "zoneId", zoneId); + ReflectionTestUtils.setField(cmd, "diskOfferingId", rootDiskOfferingId); + ReflectionTestUtils.setField(cmd, "overrideDiskOfferingId", null); + + ServiceOfferingVO serviceOffering = mock(ServiceOfferingVO.class); + DiskOfferingVO rootDiskOffering = mock(DiskOfferingVO.class); + when(_serviceOfferingDao.findById(serviceOfferingId)).thenReturn(serviceOffering); + when(rootDiskOffering.getDiskSize()).thenReturn(10 * 1024 * 1024 * 1024L); + when(diskOfferingDao.findById(rootDiskOfferingId)).thenReturn(rootDiskOffering); + + Map diskDetails = new HashMap<>(); + diskDetails.put(ApiConstants.DISK_OFFERING_ID, "disk-offering-uuid"); + diskDetails.put(ApiConstants.DEVICE_ID, "1"); + diskDetails.put(ApiConstants.SIZE, "5"); + diskDetails.put(ApiConstants.MIN_IOPS, "1000"); + diskDetails.put(ApiConstants.MAX_IOPS, "5000"); + Map> disksDetails = new HashMap<>(); + disksDetails.put(0, diskDetails); + ReflectionTestUtils.setField(cmd, "dataDisksDetails", disksDetails); + DiskOffering diskOffering = mock(DiskOffering.class); + when(diskOffering.isCustomized()).thenReturn(true); + when(diskOffering.isCustomizedIops()).thenReturn(true); + when(entityManager.findByUuid(DiskOffering.class, "disk-offering-uuid")).thenReturn(diskOffering); + + BackupVO backup = mock(BackupVO.class); + when(backup.getZoneId()).thenReturn(zoneId); + when(backup.getVmId()).thenReturn(vmId); + when(backupDao.findById(backupId)).thenReturn(backup); + + UserVmVO userVmVO = new UserVmVO(); + userVmVO.setTemplateId(isoId); + when(userVmDao.findByIdIncludingRemoved(vmId)).thenReturn(userVmVO); + VMTemplateVO iso = mock(VMTemplateVO.class); + when(iso.getFormat()).thenReturn(Storage.ImageFormat.ISO); + when(templateDao.findById(isoId)).thenReturn(iso); + VmDiskInfo rootVmDiskInfo = new VmDiskInfo(diskOffering, 10L, 1000L, 2000L); + when(backupManager.getRootDiskInfoFromBackup(backup)).thenReturn(rootVmDiskInfo); + Mockito.when(backupManager.canCreateInstanceFromBackup(backupId)).thenReturn(true); + + Mockito.doReturn(userVmVoMock).when(userVmManagerImpl).createAdvancedVirtualMachine(any(), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), + any(), any(), any(), any(), eq(true), any(), any(), any(), any()); + + UserVm result = userVmManagerImpl.allocateVMFromBackup(cmd); + + assertNotNull(result); + Mockito.verify(backupDao).findById(backupId); + Mockito.verify(userVmManagerImpl).createAdvancedVirtualMachine(any(), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), + any(), any(), any(), any(), eq(true), any(), any(), any(), any()); + } + + @Test + public void testAllocateVMFromBackupUsingBackupValuesWithISO() throws InsufficientCapacityException, ResourceAllocationException, ResourceUnavailableException { + Long backupId = 5L; + Long isoId = 9L; + + CreateVMFromBackupCmd cmd = mock(CreateVMFromBackupCmd.class); + when(cmd.getZoneId()).thenReturn(zoneId); + when(cmd.getBackupId()).thenReturn(backupId); + when(cmd.getEntityOwnerId()).thenReturn(accountId); + when(cmd.getServiceOfferingId()).thenReturn(null); + when(cmd.getDiskOfferingId()).thenReturn(null); + when(cmd.getTemplateId()).thenReturn(null); + when(cmd.getHostId()).thenReturn(null); + when(cmd.getNetworkIds()).thenReturn(null); + when(cmd.getIpToNetworkMap()).thenReturn(null); + when(cmd.getDataDiskInfoList()).thenReturn(null); + + Account owner = mock(Account.class); + when(accountService.getActiveAccountById(accountId)).thenReturn(owner); + + DataCenterVO zone = mock(DataCenterVO.class); + when(_dcDao.findById(zoneId)).thenReturn(zone); + + BackupVO backup = mock(BackupVO.class); + when(backup.getZoneId()).thenReturn(zoneId); + when(backup.getVmId()).thenReturn(vmId); + when(backup.getDetail(ApiConstants.SERVICE_OFFERING_ID)).thenReturn("service-offering-uuid"); + when(backupDao.findById(backupId)).thenReturn(backup); + + UserVmVO userVmVO = new UserVmVO(); + when(userVmDao.findByIdIncludingRemoved(vmId)).thenReturn(userVmVO); + VMTemplateVO iso = mock(VMTemplateVO.class); + when(iso.getFormat()).thenReturn(Storage.ImageFormat.ISO); + when(backup.getDetail(ApiConstants.TEMPLATE_ID)).thenReturn("iso-uuid"); + when(templateDao.findByUuid("iso-uuid")).thenReturn(iso); + + ServiceOfferingVO serviceOffering = mock(ServiceOfferingVO.class); + DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); + when(backup.getDetail(ApiConstants.SERVICE_OFFERING_ID)).thenReturn("service-offering-uuid"); + when(_serviceOfferingDao.findByUuid("service-offering-uuid")).thenReturn(serviceOffering); + + VmDiskInfo rootVmDiskInfo = new VmDiskInfo(diskOffering, 10L, 1000L, 2000L); + when(backupManager.getRootDiskInfoFromBackup(backup)).thenReturn(rootVmDiskInfo); + + NetworkVO network1 = mock(NetworkVO.class); + NetworkVO network2 = mock(NetworkVO.class); + when(backupManager.getDataDiskInfoListFromBackup(backup)).thenReturn(List.of(new VmDiskInfo(diskOffering, 10L, 1000L, 2000L))); + Mockito.when(backupManager.canCreateInstanceFromBackup(backupId)).thenReturn(true); + + Mockito.doReturn(userVmVoMock).when(userVmManagerImpl).createAdvancedVirtualMachine(any(), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), eq(false), any(), any(), any(), + any(), any(), any(), any(), eq(false), any(), any(), any(), any()); + + UserVm result = userVmManagerImpl.allocateVMFromBackup(cmd); + + assertNotNull(result); + Mockito.verify(userVmManagerImpl).createAdvancedVirtualMachine(any(), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), eq(false), any(), any(), any(), + any(), any(), any(), any(), eq(false), any(), any(), any(), any()); + } + + @Test + public void testRestoreVMFromBackup() throws ResourceUnavailableException, InsufficientCapacityException, ResourceAllocationException { + Long backupId = 5L; + Long templateId = 6L; + + CreateVMFromBackupCmd cmd = mock(CreateVMFromBackupCmd.class); + when(cmd.getBackupId()).thenReturn(backupId); + when(cmd.getStartVm()).thenReturn(true); + when(cmd.getEntityId()).thenReturn(vmId); + + UserVmVO vm = mock(UserVmVO.class); + when(vm.getId()).thenReturn(vmId); + when(vm.getState()).thenReturn(VirtualMachine.State.Running); + when(vm.getTemplateId()).thenReturn(templateId); + + when(backupManager.restoreBackupToVM(backupId, vmId)).thenReturn(true); + + Map params = new HashMap<>(); + Pair> vmPair = new Pair<>(vm, params); + doReturn(vmPair).when(userVmManagerImpl).startVirtualMachine(anyLong(), isNull(), isNull(), isNull(), anyMap(), isNull()); + doReturn(vmPair).when(userVmManagerImpl).startVirtualMachine(anyLong(), isNull(), isNull(), anyLong(), anyMap(), isNull()); + when(userVmDao.findById(vmId)).thenReturn(vm); + when(templateDao.findByIdIncludingRemoved(templateId)).thenReturn(mock(VMTemplateVO.class)); + when(userVmManagerImpl.stopVirtualMachine(anyLong(), anyLong())).thenReturn(true); + + UserVm result = userVmManagerImpl.restoreVMFromBackup(cmd); + + assertNotNull(result); + assertEquals(vm, result); + Mockito.verify(backupManager).restoreBackupToVM(backupId, vmId); + } + + @Test + public void testDestroyVm() throws ResourceUnavailableException { + Long volumeId = 4L; + Long accountId = 5L; + Long userId = 6L; + boolean expunge = true; + + ReflectionTestUtils.setField(userVmManagerImpl, "_uuidMgr", uuidMgr); + CallContext callContext = mock(CallContext.class); + Account callingAccount = mock(Account.class); + when(callingAccount.getId()).thenReturn(accountId); + when(callContext.getCallingAccount()).thenReturn(callingAccount); + when(accountManager.isAdmin(callingAccount.getId())).thenReturn(true); + doNothing().when(accountManager).checkApiAccess(callingAccount, BaseCmd.getCommandNameByClass(ExpungeVMCmd.class)); + try (MockedStatic mockedCallContext = Mockito.mockStatic(CallContext.class)) { + mockedCallContext.when(CallContext::current).thenReturn(callContext); + mockedCallContext.when(() -> CallContext.register(callContext, ApiCommandResourceType.Volume)).thenReturn(callContext); + + DestroyVMCmd cmd = mock(DestroyVMCmd.class); + when(cmd.getId()).thenReturn(vmId); + when(cmd.getExpunge()).thenReturn(expunge); + List volumeIds = List.of(volumeId); + when(cmd.getVolumeIds()).thenReturn(volumeIds); + + UserVmVO vm = mock(UserVmVO.class); + when(vm.getId()).thenReturn(vmId); + when(vm.getState()).thenReturn(VirtualMachine.State.Running); + when(vm.getUuid()).thenReturn("vm-uuid"); + when(vm.getUserVmType()).thenReturn("User"); + when(userVmDao.findById(vmId)).thenReturn(vm); + + VolumeVO vol = Mockito.mock(VolumeVO.class); + when(vol.getInstanceId()).thenReturn(vmId); + when(vol.getId()).thenReturn(volumeId); + when(vol.getVolumeType()).thenReturn(Volume.Type.DATADISK); + when(volumeDaoMock.findById(volumeId)).thenReturn(vol); + + List dataVolumes = new ArrayList<>(); + when(volumeDaoMock.findByInstanceAndType(vmId, Volume.Type.DATADISK)).thenReturn(dataVolumes); + + when(volumeApiService.destroyVolume(volumeId, CallContext.current().getCallingAccount(), expunge, false)).thenReturn(vol); + + doReturn(vm).when(userVmManagerImpl).stopVirtualMachine(anyLong(), anyBoolean()); + doReturn(vm).when(userVmManagerImpl).destroyVm(vmId, expunge); + doReturn(true).when(userVmManagerImpl).expunge(vm); + + try (MockedStatic mockedUsageEventUtils = Mockito.mockStatic(UsageEventUtils.class)) { + + UserVm result = userVmManagerImpl.destroyVm(cmd); + + assertNotNull(result); + assertEquals(vm, result); + Mockito.verify(userVmManagerImpl).stopVirtualMachine(vmId, false); + Mockito.verify(backupManager).checkAndRemoveBackupOfferingBeforeExpunge(vm); + } + } + } @Test(expected = InvalidParameterValueException.class) public void testValidateLeasePropertiesInvalidDuration() { @@ -3446,7 +3864,7 @@ public void createVirtualMachineWithExistingVolume() throws ResourceUnavailableE when(_dcMock.isLocalStorageEnabled()).thenReturn(false); when(_dcMock.getNetworkType()).thenReturn(DataCenter.NetworkType.Basic); Mockito.doReturn(userVmVoMock).when(userVmManagerImpl).createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(), - any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any()); @@ -3484,7 +3902,7 @@ public void createVirtualMachineWithExistingSnapshot() throws ResourceUnavailabl when(_dcMock.isLocalStorageEnabled()).thenReturn(false); when(_dcMock.getNetworkType()).thenReturn(DataCenter.NetworkType.Basic); Mockito.doReturn(userVmVoMock).when(userVmManagerImpl).createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(), - any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any()); diff --git a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java index 7a3576902086..931206737206 100644 --- a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java @@ -1128,6 +1128,11 @@ public boolean handleCksIsoOnNetworkVirtualRouter(Long virtualRouterId, boolean return false; } + @Override + public IpAddresses getIpAddressesFromIps(String ipAddress, String ip6Address, String macAddress) { + return null; + } + @Override public void expungeLbVmRefs(List vmIds, Long batchSize) { } diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index 609635ef3abf..61ae6cd018c7 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java @@ -16,18 +16,41 @@ // under the License. package org.apache.cloudstack.backup; +import com.cloud.api.query.dao.UserVmJoinDao; +import com.cloud.api.query.vo.UserVmJoinVO; import com.cloud.alert.AlertManager; +import com.cloud.capacity.CapacityVO; import com.cloud.configuration.Resource; +import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.domain.Domain; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.network.Network; +import com.cloud.network.NetworkService; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.offering.DiskOffering; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.DiskOfferingVO; import com.cloud.exception.ResourceAllocationException; +import com.cloud.storage.Storage; +import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; @@ -35,27 +58,36 @@ import com.cloud.user.DomainManager; import com.cloud.user.ResourceLimitService; import com.cloud.user.User; -import com.cloud.user.UserVO; +import com.cloud.user.dao.AccountDao; import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; +import com.cloud.vm.VMInstanceDetailVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmDiskInfo; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.dao.VMInstanceDao; import com.google.gson.Gson; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; +import org.apache.cloudstack.api.command.user.backup.CreateBackupCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.DeleteBackupScheduleCmd; +import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupDetailsDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.backup.dao.BackupScheduleDao; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.junit.After; import org.junit.Assert; @@ -76,6 +108,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.TimeZone; import java.util.UUID; @@ -85,10 +118,12 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.atLeastOnce; @RunWith(MockitoJUnitRunner.class) public class BackupManagerTest { @@ -99,6 +134,9 @@ public class BackupManagerTest { @Mock BackupOfferingDao backupOfferingDao; + @Mock + BackupDetailsDao backupDetailsDao; + @Mock BackupProvider backupProvider; @@ -162,7 +200,38 @@ public class BackupManagerTest { @Mock private DeleteBackupScheduleCmd deleteBackupScheduleCmdMock; - private UserVO user; + @Mock + DiskOfferingDao diskOfferingDao; + + @Mock + ServiceOfferingDao serviceOfferingDao; + + @Mock + VMTemplateDao vmTemplateDao; + + @Mock + UserVmJoinDao userVmJoinDao; + + @Mock + PrimaryDataStoreDao primaryDataStoreDao; + + @Mock + HostDao hostDao; + + @Mock + private NetworkDao networkDao; + + @Mock + private NetworkService networkService; + + @Mock + private VMInstanceDetailsDao vmInstanceDetailsDao; + + @Mock + AccountDao accountDao; + + @Mock + DomainDao domainDao; private Gson gson; @@ -196,6 +265,12 @@ public void setup() throws Exception { return true; }); + backupProvider = mock(BackupProvider.class); + when(backupProvider.getName()).thenReturn("testbackupprovider"); + Map backupProvidersMap = new HashMap<>(); + backupProvidersMap.put(backupProvider.getName().toLowerCase(), backupProvider); + ReflectionTestUtils.setField(backupManager, "backupProvidersMap", backupProvidersMap); + Account account = mock(Account.class); User user = mock(User.class); CallContext.register(user, account); @@ -210,6 +285,20 @@ public void tearDown() throws Exception { CallContext.unregister(); } + private void overrideBackupFrameworkConfigValue() { + ConfigKey configKey = BackupManager.BackupFrameworkEnabled; + this.configDepotImpl = (ConfigDepotImpl) ReflectionTestUtils.getField(configKey, "s_depot"); + ConfigDepotImpl configDepot = Mockito.mock(ConfigDepotImpl.class); + Mockito.when(configDepot.getConfigStringValue(Mockito.eq(BackupManager.BackupFrameworkEnabled.key()), + Mockito.eq(ConfigKey.Scope.Global), Mockito.isNull())).thenReturn("true"); + Mockito.when(configDepot.getConfigStringValue(Mockito.eq(BackupManager.BackupFrameworkEnabled.key()), + Mockito.eq(ConfigKey.Scope.Zone), Mockito.anyLong())).thenReturn("true"); + Mockito.when(configDepot.getConfigStringValue(Mockito.eq(BackupManager.BackupProviderPlugin.key()), + Mockito.eq(ConfigKey.Scope.Zone), Mockito.anyLong())).thenReturn("testbackupprovider"); + ReflectionTestUtils.setField(configKey, "s_depot", configDepot); + updatedConfigKeyDepot = true; + } + @Test public void testExceptionWhenUpdateWithNullId() { try { @@ -224,7 +313,7 @@ public void testExceptionWhenUpdateWithNullId() { } } - @Test (expected = InvalidParameterValueException.class) + @Test(expected = InvalidParameterValueException.class) public void testExceptionWhenUpdateWithNonExistentId() { Long id = 123l; @@ -234,7 +323,7 @@ public void testExceptionWhenUpdateWithNonExistentId() { backupManager.updateBackupOffering(cmd); } - @Test (expected = ServerApiException.class) + @Test(expected = ServerApiException.class) public void testExceptionWhenUpdateWithoutChanges() { UpdateBackupOfferingCmd cmd = Mockito.spy(UpdateBackupOfferingCmd.class); when(cmd.getName()).thenReturn(null); @@ -265,96 +354,117 @@ public void testUpdateBackupOfferingSuccess() { @Test public void restoreBackedUpVolumeTestHostIpAndDatastoreUuid() { BackupVO backupVO = new BackupVO(); - VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + VMInstanceVO vm = mock(VMInstanceVO.class); String volumeUuid = "5f4ed903-ac23-4f8a-b595-69c73c40593f"; String vmName = "i-2-3-VM"; VirtualMachine.State vmState = VirtualMachine.State.Running; Mockito.when(vm.getName()).thenReturn(vmName); Mockito.when(vm.getState()).thenReturn(vmState); - Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); + Pair vmNameAndState = new Pair<>(vmName, vmState); + + Backup.VolumeInfo volumeInfo = mock(Backup.VolumeInfo.class); + when(volumeInfo.getUuid()).thenReturn(volumeUuid); - Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), - Mockito.eq("127.0.0.1"), Mockito.eq("e9804933-8609-4de3-bccc-6278072a496c"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + doReturn(new Pair(Boolean.TRUE, "Success")) + .when(backupProvider).restoreBackedUpVolume(any(Backup.class), any(Backup.VolumeInfo.class), + any(String.class), any(String.class), any(Pair.class)); + + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeInfo, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success", restoreBackedUpVolume.second()); - Mockito.verify(backupProvider, times(1)).restoreBackedUpVolume(Mockito.any(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), any(Pair.class)); + verify(backupProvider, atLeastOnce()).restoreBackedUpVolume(any(Backup.class), any(Backup.VolumeInfo.class), + any(String.class), any(String.class), any(Pair.class)); } @Test public void restoreBackedUpVolumeTestHostIpAndDatastoreName() { BackupVO backupVO = new BackupVO(); - VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + VMInstanceVO vm = mock(VMInstanceVO.class); String volumeUuid = "5f4ed903-ac23-4f8a-b595-69c73c40593f"; String vmName = "i-2-3-VM"; VirtualMachine.State vmState = VirtualMachine.State.Running; Mockito.when(vm.getName()).thenReturn(vmName); Mockito.when(vm.getState()).thenReturn(vmState); - Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); - Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), - Mockito.eq("127.0.0.1"), Mockito.eq("datastore-name"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success2")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + Pair vmNameAndState = new Pair<>(vmName, vmState); + + Backup.VolumeInfo volumeInfo = mock(Backup.VolumeInfo.class); + when(volumeInfo.getUuid()).thenReturn(volumeUuid); + + doReturn(new Pair(Boolean.TRUE, "Success2")) + .when(backupProvider).restoreBackedUpVolume(any(Backup.class), any(Backup.VolumeInfo.class), + any(String.class), any(String.class), any(Pair.class)); + + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeInfo, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success2", restoreBackedUpVolume.second()); - Mockito.verify(backupProvider, times(2)).restoreBackedUpVolume(Mockito.any(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), any(Pair.class)); + verify(backupProvider, atLeastOnce()).restoreBackedUpVolume(any(Backup.class), any(Backup.VolumeInfo.class), + any(String.class), any(String.class), any(Pair.class)); } @Test public void restoreBackedUpVolumeTestHostNameAndDatastoreUuid() { BackupVO backupVO = new BackupVO(); - VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + VMInstanceVO vm = mock(VMInstanceVO.class); String volumeUuid = "5f4ed903-ac23-4f8a-b595-69c73c40593f"; String vmName = "i-2-3-VM"; VirtualMachine.State vmState = VirtualMachine.State.Running; Mockito.when(vm.getName()).thenReturn(vmName); Mockito.when(vm.getState()).thenReturn(vmState); - Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); + Pair vmNameAndState = new Pair<>(vmName, vmState); + + Backup.VolumeInfo volumeInfo = mock(Backup.VolumeInfo.class); + when(volumeInfo.getUuid()).thenReturn(volumeUuid); + + doReturn(new Pair(Boolean.TRUE, "Success3")) + .when(backupProvider).restoreBackedUpVolume(any(Backup.class), any(Backup.VolumeInfo.class), + any(String.class), any(String.class), any(Pair.class)); - Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), - Mockito.eq("hostname"), Mockito.eq("e9804933-8609-4de3-bccc-6278072a496c"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success3")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeInfo, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success3", restoreBackedUpVolume.second()); - Mockito.verify(backupProvider, times(3)).restoreBackedUpVolume(Mockito.any(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), any(Pair.class)); + verify(backupProvider, atLeastOnce()).restoreBackedUpVolume(any(Backup.class), any(Backup.VolumeInfo.class), + any(String.class), any(String.class), any(Pair.class)); } @Test public void restoreBackedUpVolumeTestHostAndDatastoreName() { BackupVO backupVO = new BackupVO(); - VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + VMInstanceVO vm = mock(VMInstanceVO.class); String volumeUuid = "5f4ed903-ac23-4f8a-b595-69c73c40593f"; String vmName = "i-2-3-VM"; VirtualMachine.State vmState = VirtualMachine.State.Running; Mockito.when(vm.getName()).thenReturn(vmName); Mockito.when(vm.getState()).thenReturn(vmState); - Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); + Pair vmNameAndState = new Pair<>(vmName, vmState); + + Backup.VolumeInfo volumeInfo = mock(Backup.VolumeInfo.class); + when(volumeInfo.getUuid()).thenReturn(volumeUuid); - Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), - Mockito.eq("hostname"), Mockito.eq("datastore-name"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success4")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + doReturn(new Pair(Boolean.TRUE, "Success4")) + .when(backupProvider).restoreBackedUpVolume(any(Backup.class), any(Backup.VolumeInfo.class), + any(String.class), any(String.class), any(Pair.class)); + + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeInfo, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success4", restoreBackedUpVolume.second()); - Mockito.verify(backupProvider, times(4)).restoreBackedUpVolume(Mockito.any(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), any(Pair.class)); + verify(backupProvider, atLeastOnce()).restoreBackedUpVolume(any(Backup.class), any(Backup.VolumeInfo.class), + any(String.class), any(String.class), any(Pair.class)); } @Test public void tryRestoreVMTestRestoreSucceeded() throws NoTransitionException { - BackupOffering offering = Mockito.mock(BackupOffering.class); - VolumeVO volumeVO = Mockito.mock(VolumeVO.class); - VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); - BackupVO backup = Mockito.mock(BackupVO.class); + BackupOffering offering = mock(BackupOffering.class); + VolumeVO volumeVO = mock(VolumeVO.class); + VMInstanceVO vm = mock(VMInstanceVO.class); + BackupVO backup = mock(BackupVO.class); try (MockedStatic utils = Mockito.mockStatic(ActionEventUtils.class)) { Mockito.when(ActionEventUtils.onStartedActionEvent(Mockito.anyLong(), Mockito.anyLong(), @@ -379,10 +489,10 @@ public void tryRestoreVMTestRestoreSucceeded() throws NoTransitionException { @Test public void tryRestoreVMTestRestoreFails() throws NoTransitionException { - BackupOffering offering = Mockito.mock(BackupOffering.class); - VolumeVO volumeVO = Mockito.mock(VolumeVO.class); - VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); - BackupVO backup = Mockito.mock(BackupVO.class); + BackupOffering offering = mock(BackupOffering.class); + VolumeVO volumeVO = mock(VolumeVO.class); + VMInstanceVO vm = mock(VMInstanceVO.class); + BackupVO backup = mock(BackupVO.class); try (MockedStatic utils = Mockito.mockStatic(ActionEventUtils.class)) { Mockito.when(ActionEventUtils.onStartedActionEvent(Mockito.anyLong(), Mockito.anyLong(), @@ -411,20 +521,6 @@ public void tryRestoreVMTestRestoreFails() throws NoTransitionException { } } - private void overrideBackupFrameworkConfigValue() { - ConfigKey configKey = BackupManager.BackupFrameworkEnabled; - this.configDepotImpl = (ConfigDepotImpl) ReflectionTestUtils.getField(configKey, "s_depot"); - ConfigDepotImpl configDepot = Mockito.mock(ConfigDepotImpl.class); - Mockito.when(configDepot.getConfigStringValue(Mockito.eq(BackupManager.BackupFrameworkEnabled.key()), - Mockito.eq(ConfigKey.Scope.Global), Mockito.isNull())).thenReturn("true"); - Mockito.when(configDepot.getConfigStringValue(Mockito.eq(BackupManager.BackupFrameworkEnabled.key()), - Mockito.eq(ConfigKey.Scope.Zone), Mockito.anyLong())).thenReturn("true"); - Mockito.when(configDepot.getConfigStringValue(Mockito.eq(BackupManager.BackupProviderPlugin.key()), - Mockito.eq(ConfigKey.Scope.Zone), Mockito.anyLong())).thenReturn("testbackupprovider"); - ReflectionTestUtils.setField(configKey, "s_depot", configDepot); - updatedConfigKeyDepot = true; - } - @Test public void testConfigureBackupSchedule() { Long vmId = 1L; @@ -439,6 +535,7 @@ public void testConfigureBackupSchedule() { when(cmd.getIntervalType()).thenReturn(DateUtil.IntervalType.DAILY); when(cmd.getMaxBackups()).thenReturn(8); when(cmd.getSchedule()).thenReturn("00:00:00"); + when(cmd.getQuiesceVM()).thenReturn(null); VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); when(vmInstanceDao.findById(vmId)).thenReturn(vm); @@ -514,7 +611,9 @@ public void createBackupTestCreateScheduledBackup() throws ResourceAllocationExc Long backupOfferingId = 4L; Long accountId = 5L; Long backupId = 6L; + Long oldestBackupId = 7L; Long newBackupSize = 1000000000L; + Long oldBackupSize = 400000000L; when(vmInstanceDao.findById(vmId)).thenReturn(vmInstanceVOMock); when(vmInstanceVOMock.getDataCenterId()).thenReturn(zoneId); @@ -524,7 +623,7 @@ public void createBackupTestCreateScheduledBackup() throws ResourceAllocationExc overrideBackupFrameworkConfigValue(); when(backupOfferingDao.findById(backupOfferingId)).thenReturn(backupOfferingVOMock); when(backupOfferingVOMock.isUserDrivenBackupAllowed()).thenReturn(true); - when(backupOfferingVOMock.getProvider()).thenReturn("test"); + when(backupOfferingVOMock.getProvider()).thenReturn("testbackupprovider"); Mockito.doReturn(scheduleId).when(backupManager).getBackupScheduleId(asyncJobVOMock); @@ -534,31 +633,45 @@ public void createBackupTestCreateScheduledBackup() throws ResourceAllocationExc when(backupScheduleDao.findById(scheduleId)).thenReturn(schedule); when(schedule.getMaxBackups()).thenReturn(2); + VolumeVO volume = mock(VolumeVO.class); + when(volumeDao.findByInstance(vmId)).thenReturn(List.of(volume)); + when(volume.getState()).thenReturn(Volume.State.Ready); + when(volumeApiService.getVolumePhysicalSize(null, null, null)).thenReturn(newBackupSize); + BackupProvider backupProvider = mock(BackupProvider.class); Backup backup = mock(Backup.class); when(backup.getId()).thenReturn(backupId); when(backup.getSize()).thenReturn(newBackupSize); - when(backupProvider.getName()).thenReturn("test"); - when(backupProvider.takeBackup(vmInstanceVOMock)).thenReturn(new Pair<>(true, backup)); + when(backupProvider.getName()).thenReturn("testbackupprovider"); + when(backupProvider.takeBackup(vmInstanceVOMock, null)).thenReturn(new Pair<>(true, backup)); Map backupProvidersMap = new HashMap<>(); backupProvidersMap.put(backupProvider.getName().toLowerCase(), backupProvider); ReflectionTestUtils.setField(backupManager, "backupProvidersMap", backupProvidersMap); BackupVO backupVO = mock(BackupVO.class); when(backupVO.getId()).thenReturn(backupId); - BackupVO oldestBackupVO = mock(BackupVO.class);; + BackupVO oldestBackupVO = mock(BackupVO.class); when(backupDao.findById(backupId)).thenReturn(backupVO); List backups = new ArrayList<>(List.of(oldestBackupVO)); when(backupDao.listBySchedule(scheduleId)).thenReturn(backups); + CreateBackupCmd cmd = Mockito.mock(CreateBackupCmd.class); + when(cmd.getVmId()).thenReturn(vmId); + when(cmd.getName()).thenReturn("new-backup1"); + when(cmd.getQuiesceVM()).thenReturn(null); + try (MockedStatic ignored = Mockito.mockStatic(ActionEventUtils.class)) { Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.anyLong(), Mockito.anyString())).thenReturn(1L); - assertTrue(backupManager.createBackup(vmId, asyncJobVOMock)); + assertTrue(backupManager.createBackup(cmd, asyncJobVOMock)); + + Mockito.verify(resourceLimitMgr, times(1)).checkResourceLimit(accountVOMock, Resource.ResourceType.backup); + Mockito.verify(resourceLimitMgr, times(1)).checkResourceLimit(accountVOMock, Resource.ResourceType.backup_storage, newBackupSize); + Mockito.verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup); Mockito.verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup_storage, newBackupSize); Mockito.verify(backupDao, times(1)).update(backupVO.getId(), backupVO); @@ -583,6 +696,46 @@ public void createBackupTestResourceLimitReached() throws ResourceAllocationExce BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); when(backupOfferingDao.findById(backupOfferingId)).thenReturn(offering); when(offering.isUserDrivenBackupAllowed()).thenReturn(true); + when(offering.getProvider()).thenReturn("testbackupprovider"); + + Account account = Mockito.mock(Account.class); + when(accountManager.getAccount(accountId)).thenReturn(account); + Mockito.doThrow(new ResourceAllocationException("", Resource.ResourceType.backup)).when(resourceLimitMgr).checkResourceLimit(account, Resource.ResourceType.backup); + + CreateBackupCmd cmd = Mockito.mock(CreateBackupCmd.class); + when(cmd.getVmId()).thenReturn(vmId); + when(cmd.getQuiesceVM()).thenReturn(null); + + String jobParams = "{}"; + when(asyncJobVOMock.getCmdInfo()).thenReturn(jobParams); + when(asyncJobVOMock.getId()).thenReturn(1L); + + backupManager.createBackup(cmd, asyncJobVOMock); + + String msg = "Backup storage space resource limit exceeded for account id : " + accountId + ". Failed to create backup"; + Mockito.verify(alertManagerMock, times(1)).sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, "Backup resource limit exceeded for account id : " + accountId + + ". Failed to create backups; please use updateResourceLimit to increase the limit"); + } + + @Test (expected = ResourceAllocationException.class) + public void testCreateBackupStorageLimitReached() throws ResourceAllocationException { + Long vmId = 1L; + Long zoneId = 2L; + Long scheduleId = 3L; + Long backupOfferingId = 4L; + Long accountId = 5L; + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vmInstanceDao.findById(vmId)).thenReturn(vm); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getBackupOfferingId()).thenReturn(backupOfferingId); + when(vm.getAccountId()).thenReturn(accountId); + + overrideBackupFrameworkConfigValue(); + BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); + when(backupOfferingDao.findById(backupOfferingId)).thenReturn(offering); + when(offering.isUserDrivenBackupAllowed()).thenReturn(true); + when(offering.getProvider()).thenReturn("testbackupprovider"); Mockito.doReturn(scheduleId).when(backupManager).getBackupScheduleId(asyncJobVOMock); @@ -590,7 +743,15 @@ public void createBackupTestResourceLimitReached() throws ResourceAllocationExce when(accountManager.getAccount(accountId)).thenReturn(account); Mockito.doThrow(new ResourceAllocationException("", Resource.ResourceType.backup_storage)).when(resourceLimitMgr).checkResourceLimit(account, Resource.ResourceType.backup_storage, 0L); - backupManager.createBackup(vmId, asyncJobVOMock); + CreateBackupCmd cmd = Mockito.mock(CreateBackupCmd.class); + when(cmd.getVmId()).thenReturn(vmId); + when(cmd.getQuiesceVM()).thenReturn(null); + + backupManager.createBackup(cmd, asyncJobVOMock); + + String msg = "Backup storage space resource limit exceeded for account id : " + accountId + ". Failed to create backup"; + Mockito.verify(alertManagerMock, times(1)).sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, "Backup storage space resource limit exceeded for account id : " + accountId + + ". Failed to create backups; please use updateResourceLimit to increase the limit"); } @Test @@ -603,7 +764,7 @@ public void testBackupSyncTask() { Long backup1Size = 1 * Resource.ResourceType.bytesToGiB; Long backup2Size = 2 * Resource.ResourceType.bytesToGiB; Long newBackupSize = 3 * Resource.ResourceType.bytesToGiB; - Long metricSize = 4 * Resource.ResourceType.bytesToGiB; + Long restorePointSize = 4 * Resource.ResourceType.bytesToGiB; overrideBackupFrameworkConfigValue(); @@ -619,23 +780,23 @@ public void testBackupSyncTask() { VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); when(vm.getId()).thenReturn(vmId); when(vm.getAccountId()).thenReturn(accountId); - when(vmInstanceDao.listByZoneWithBackups(dataCenterId, null)).thenReturn(List.of(vm)); - Backup.Metric metric = new Backup.Metric(metricSize, null); - Map metricMap = new HashMap<>(); - metricMap.put(vm, metric); - when(backupProvider.getBackupMetrics(Mockito.anyLong(), Mockito.anyList())).thenReturn(metricMap); - - Backup.RestorePoint restorePoint1 = new Backup.RestorePoint(restorePoint1ExternalId, DateUtil.now(), "Root"); - Backup.RestorePoint restorePoint2 = new Backup.RestorePoint("12345", DateUtil.now(), "Root"); + List vmIds = List.of(vmId); + when(backupDao.listVmIdsWithBackupsInZone(dataCenterId)).thenReturn(vmIds); + when(vmInstanceDao.listByZoneAndBackupOffering(dataCenterId, null)).thenReturn(List.of(vm)); + + Backup.RestorePoint restorePoint1 = new Backup.RestorePoint(restorePoint1ExternalId, DateUtil.now(), "Full", restorePointSize, 0L); + Backup.RestorePoint restorePoint2 = new Backup.RestorePoint("12345", DateUtil.now(), "Full", restorePointSize, 0L); List restorePoints = new ArrayList<>(List.of(restorePoint1, restorePoint2)); when(backupProvider.listRestorePoints(vm)).thenReturn(restorePoints); BackupVO backupInDb1 = new BackupVO(); backupInDb1.setSize(backup1Size); + backupInDb1.setAccountId(accountId); backupInDb1.setExternalId(restorePoint1ExternalId); BackupVO backupInDb2 = new BackupVO(); backupInDb2.setSize(backup2Size); + backupInDb2.setAccountId(accountId); backupInDb2.setExternalId(null); ReflectionTestUtils.setField(backupInDb2, "id", backup2Id); when(backupDao.findById(backup2Id)).thenReturn(backupInDb2); @@ -644,7 +805,7 @@ public void testBackupSyncTask() { BackupVO newBackupEntry = new BackupVO(); newBackupEntry.setSize(newBackupSize); - when(backupProvider.createNewBackupEntryForRestorePoint(restorePoint2, vm, metric)).thenReturn(newBackupEntry); + when(backupProvider.createNewBackupEntryForRestorePoint(restorePoint2, vm)).thenReturn(newBackupEntry); try (MockedStatic ignored = Mockito.mockStatic(ActionEventUtils.class)) { Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(), @@ -658,8 +819,8 @@ public void testBackupSyncTask() { backupSyncTask.runInContext(); verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup_storage, backup1Size); - verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup_storage, metricSize); - Assert.assertEquals(backupInDb1.getSize(), metricSize); + verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup_storage, restorePointSize); + Assert.assertEquals(backupInDb1.getSize(), restorePointSize); verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup); verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup_storage, newBackupSize); @@ -762,6 +923,689 @@ public void deleteBackupScheduleTestDeleteSpecificScheduleWhenItsIdIsSpecified() assertTrue(success); } + @Test + public void testGetBackupDetailsFromVM() { + Long vmId = 1L; + VirtualMachine vm = mock(VirtualMachine.class); + when(vm.getServiceOfferingId()).thenReturn(1L); + when(vm.getTemplateId()).thenReturn(2L); + when(vm.getId()).thenReturn(vmId); + + ServiceOfferingVO serviceOffering = mock(ServiceOfferingVO.class); + when(serviceOffering.getUuid()).thenReturn("service-offering-uuid"); + when(serviceOfferingDao.findById(1L)).thenReturn(serviceOffering); + VMTemplateVO template = mock(VMTemplateVO.class); + when(template.getUuid()).thenReturn("template-uuid"); + when(vmTemplateDao.findById(2L)).thenReturn(template); + + VMInstanceDetailVO vmInstanceDetail = mock(VMInstanceDetailVO.class); + when(vmInstanceDetail.getName()).thenReturn("mocked-detail-name"); + when(vmInstanceDetail.getValue()).thenReturn("mocked-detail-value"); + List vmDetails = Collections.singletonList(vmInstanceDetail); + when(vmInstanceDetailsDao.listDetails(vmId)).thenReturn(vmDetails); + + UserVmJoinVO userVmJoinVO = mock(UserVmJoinVO.class); + when(userVmJoinVO.getNetworkUuid()).thenReturn("mocked-network-uuid"); + List userVmJoinVOs = Collections.singletonList(userVmJoinVO); + when(userVmJoinDao.searchByIds(vmId)).thenReturn(userVmJoinVOs); + + Map details = backupManager.getBackupDetailsFromVM(vm); + + assertEquals("service-offering-uuid", details.get(ApiConstants.SERVICE_OFFERING_ID)); + assertEquals("[{\"networkid\":\"mocked-network-uuid\"}]", details.get(ApiConstants.NICS)); + assertEquals("{\"mocked-detail-name\":\"mocked-detail-value\"}", details.get(ApiConstants.VM_SETTINGS)); + } + + @Test + public void getDataDiskInfoListFromBackup() { + Long size1 = 5L * 1024 * 1024 * 1024; + Long size2 = 10L * 1024 * 1024 * 1024; + Backup backup = mock(Backup.class); + + Backup.VolumeInfo volumeInfo0 = mock(Backup.VolumeInfo.class); + when(volumeInfo0.getType()).thenReturn(Volume.Type.ROOT); + Backup.VolumeInfo volumeInfo1 = mock(Backup.VolumeInfo.class); + when(volumeInfo1.getDiskOfferingId()).thenReturn("disk-offering-uuid-1"); + when(volumeInfo1.getSize()).thenReturn(size1); + when(volumeInfo1.getMinIops()).thenReturn(100L); + when(volumeInfo1.getMaxIops()).thenReturn(300L); + when(volumeInfo1.getType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo1.getDeviceId()).thenReturn(1L); + Backup.VolumeInfo volumeInfo2 = mock(Backup.VolumeInfo.class); + when(volumeInfo2.getDiskOfferingId()).thenReturn("disk-offering-uuid-2"); + when(volumeInfo2.getSize()).thenReturn(size2); + when(volumeInfo2.getMinIops()).thenReturn(200L); + when(volumeInfo2.getMaxIops()).thenReturn(400L); + when(volumeInfo2.getType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo2.getDeviceId()).thenReturn(2L); + when(backup.getBackedUpVolumes()).thenReturn(List.of(volumeInfo0, volumeInfo1, volumeInfo2)); + + DiskOfferingVO diskOffering1 = mock(DiskOfferingVO.class); + when(diskOffering1.getUuid()).thenReturn("disk-offering-uuid-1"); + when(diskOffering1.getState()).thenReturn(DiskOffering.State.Active); + + DiskOfferingVO diskOffering2 = mock(DiskOfferingVO.class); + when(diskOffering2.getUuid()).thenReturn("disk-offering-uuid-2"); + when(diskOffering2.getState()).thenReturn(DiskOffering.State.Active); + + when(diskOfferingDao.findByUuid("disk-offering-uuid-1")).thenReturn(diskOffering1); + when(diskOfferingDao.findByUuid("disk-offering-uuid-2")).thenReturn(diskOffering2); + + List vmDiskInfoList = backupManager.getDataDiskInfoListFromBackup(backup); + + assertEquals(2, vmDiskInfoList.size()); + assertEquals("disk-offering-uuid-1", vmDiskInfoList.get(0).getDiskOffering().getUuid()); + assertEquals(Long.valueOf(5), vmDiskInfoList.get(0).getSize()); + assertEquals(Long.valueOf(1), vmDiskInfoList.get(0).getDeviceId()); + assertEquals(Long.valueOf(100), vmDiskInfoList.get(0).getMinIops()); + assertEquals(Long.valueOf(300), vmDiskInfoList.get(0).getMaxIops()); + + assertEquals("disk-offering-uuid-2", vmDiskInfoList.get(1).getDiskOffering().getUuid()); + assertEquals(Long.valueOf(10), vmDiskInfoList.get(1).getSize()); + assertEquals(Long.valueOf(2), vmDiskInfoList.get(1).getDeviceId()); + assertEquals(Long.valueOf(200), vmDiskInfoList.get(1).getMinIops()); + assertEquals(Long.valueOf(400), vmDiskInfoList.get(1).getMaxIops()); + } + + @Test + public void getDataDiskInfoListFromBackupNullIops() { + Long size = 5L * 1024 * 1024 * 1024; + Backup backup = mock(Backup.class); + Backup.VolumeInfo volumeInfo1 = mock(Backup.VolumeInfo.class); + when(volumeInfo1.getDiskOfferingId()).thenReturn("disk-offering-uuid-1"); + when(volumeInfo1.getSize()).thenReturn(size); + when(volumeInfo1.getMinIops()).thenReturn(null); + when(volumeInfo1.getMaxIops()).thenReturn(null); + when(volumeInfo1.getType()).thenReturn(Volume.Type.DATADISK); + when(volumeInfo1.getDeviceId()).thenReturn(1L); + when(backup.getBackedUpVolumes()).thenReturn(List.of(volumeInfo1)); + + DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); + when(diskOffering.getUuid()).thenReturn("disk-offering-uuid-1"); + when(diskOffering.getState()).thenReturn(DiskOffering.State.Active); + + when(diskOfferingDao.findByUuid("disk-offering-uuid-1")).thenReturn(diskOffering); + + List vmDiskInfoList = backupManager.getDataDiskInfoListFromBackup(backup); + + assertEquals(1, vmDiskInfoList.size()); + assertEquals("disk-offering-uuid-1", vmDiskInfoList.get(0).getDiskOffering().getUuid()); + assertEquals(Long.valueOf(5), vmDiskInfoList.get(0).getSize()); + assertEquals(Long.valueOf(1), vmDiskInfoList.get(0).getDeviceId()); + assertNull(vmDiskInfoList.get(0).getMinIops()); + assertNull(vmDiskInfoList.get(0).getMaxIops()); + } + + @Test (expected = InvalidParameterValueException.class) + public void testCheckVmDisksSizeAgainstBackup() { + Long sizeInBackup = 5L * 1024 * 1024 * 1024; + Long sizeInCmd = 2L; + Backup backup = mock(Backup.class); + Backup.VolumeInfo volumeInfo = mock(Backup.VolumeInfo.class); + when(volumeInfo.getDiskOfferingId()).thenReturn("disk-offering-uuid-1"); + when(volumeInfo.getSize()).thenReturn(sizeInBackup); + when(volumeInfo.getType()).thenReturn(Volume.Type.DATADISK); + when(backup.getBackedUpVolumes()).thenReturn(List.of(volumeInfo)); + + DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); + when(diskOffering.getState()).thenReturn(DiskOffering.State.Active); + when(diskOfferingDao.findByUuid("disk-offering-uuid-1")).thenReturn(diskOffering); + List vmDiskInfoList = List.of(new VmDiskInfo(diskOffering, sizeInCmd, 1L, null, null)); + + backupManager.checkVmDisksSizeAgainstBackup(vmDiskInfoList, backup); + } + + @Test + public void testGetRootDiskInfoFromBackup() { + Long size = 5L * 1024 * 1024 * 1024; + Backup backup = mock(Backup.class); + Backup.VolumeInfo volumeInfo = mock(Backup.VolumeInfo.class); + when(volumeInfo.getDiskOfferingId()).thenReturn("root-disk-offering-uuid"); + when(volumeInfo.getSize()).thenReturn(size); + when(volumeInfo.getType()).thenReturn(Volume.Type.ROOT); + when(backup.getBackedUpVolumes()).thenReturn(List.of(volumeInfo)); + + DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); + when(diskOffering.getUuid()).thenReturn("root-disk-offering-uuid"); + when(diskOfferingDao.findByUuid("root-disk-offering-uuid")).thenReturn(diskOffering); + + VmDiskInfo VmDiskInfo = backupManager.getRootDiskInfoFromBackup(backup); + + assertEquals("root-disk-offering-uuid", VmDiskInfo.getDiskOffering().getUuid()); + assertEquals(Long.valueOf(5), VmDiskInfo.getSize()); + assertEquals(null, VmDiskInfo.getDeviceId()); + } + + @Test + public void testImportBackupOffering() { + ImportBackupOfferingCmd cmd = Mockito.mock(ImportBackupOfferingCmd.class); + when(cmd.getZoneId()).thenReturn(1L); + when(cmd.getExternalId()).thenReturn("external-id"); + when(cmd.getName()).thenReturn("Test Offering"); + when(cmd.getDescription()).thenReturn("Test Description"); + when(cmd.getUserDrivenBackups()).thenReturn(true); + + overrideBackupFrameworkConfigValue(); + + when(backupOfferingDao.findByExternalId("external-id", 1L)).thenReturn(null); + when(backupOfferingDao.findByName("Test Offering", 1L)).thenReturn(null); + + BackupOfferingVO offering = new BackupOfferingVO(1L, "external-id", "testbackupprovider", "Test Offering", "Test Description", true); + when(backupOfferingDao.persist(any(BackupOfferingVO.class))).thenReturn(offering); + when(backupProvider.isValidProviderOffering(cmd.getZoneId(), cmd.getExternalId())).thenReturn(true); + + BackupOffering result = backupManager.importBackupOffering(cmd); + + assertEquals("Test Offering", result.getName()); + assertEquals("Test Description", result.getDescription()); + assertEquals(true, result.isUserDrivenBackupAllowed()); + assertEquals("external-id", result.getExternalId()); + assertEquals("testbackupprovider", result.getProvider()); + } + + @Test + public void testCreateVolumeInfoFromVolumes() { + Long diskOfferingId = 5L; + DiskOfferingVO diskOffering = Mockito.mock(DiskOfferingVO.class); + Mockito.when(diskOffering.getUuid()).thenReturn("disk-offering-uuid"); + Mockito.when(diskOfferingDao.findById(diskOfferingId)).thenReturn(diskOffering); + + List volumes = new ArrayList<>(); + VolumeVO volume1 = new VolumeVO(Volume.Type.ROOT, "vol1", 1L, 2L, 3L, + diskOfferingId, null, 1024L, null, null, null); + volume1.setUuid("uuid1"); + volume1.setPath("path1"); + volume1.setDeviceId(0L); + volume1.setVolumeType(Volume.Type.ROOT); + volumes.add(volume1); + + VolumeVO volume2 = new VolumeVO(Volume.Type.ROOT, "vol2", 1L, 2L, 3L, + diskOfferingId, null, 2048L, 1000L, 2000L, null); + volume2.setUuid("uuid2"); + volume2.setPath("path2"); + volume2.setDeviceId(1L); + volume2.setVolumeType(Volume.Type.DATADISK); + volumes.add(volume2); + + String expectedJson = "[{\"uuid\":\"uuid1\",\"type\":\"ROOT\",\"size\":1024,\"path\":\"path1\",\"deviceId\":0,\"diskOfferingId\":\"disk-offering-uuid\"},{\"uuid\":\"uuid2\",\"type\":\"DATADISK\",\"size\":2048,\"path\":\"path2\",\"deviceId\":1,\"diskOfferingId\":\"disk-offering-uuid\",\"minIops\":1000,\"maxIops\":2000}]"; + String actualJson = backupManager.createVolumeInfoFromVolumes(new ArrayList<>(volumes)); + + assertEquals(expectedJson, actualJson); + } + + @Test + public void testAssignVMToBackupOffering() { + Long vmId = 1L; + Long offeringId = 2L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + when(vm.getId()).thenReturn(vmId); + BackupOfferingVO offering = mock(BackupOfferingVO.class); + + overrideBackupFrameworkConfigValue(); + + when(vmInstanceDao.findById(vmId)).thenReturn(vm); + when(backupOfferingDao.findById(offeringId)).thenReturn(offering); + when(vm.getState()).thenReturn(VirtualMachine.State.Running); + when(vm.getDataCenterId()).thenReturn(1L); + when(vm.getBackupOfferingId()).thenReturn(null); + when(offering.getProvider()).thenReturn("testbackupprovider"); + when(backupProvider.assignVMToBackupOffering(vm, offering)).thenReturn(true); + when(vmInstanceDao.update(1L, vm)).thenReturn(true); + + try (MockedStatic ignored2 = Mockito.mockStatic(UsageEventUtils.class)) { + boolean result = backupManager.assignVMToBackupOffering(vmId, offeringId); + + assertTrue(result); + verify(vmInstanceDao, times(1)).findById(vmId); + verify(backupOfferingDao, times(1)).findById(offeringId); + verify(backupManager, times(1)).getBackupProvider("testbackupprovider"); + } + } + + @Test + public void testRemoveVMFromBackupOffering() { + Long vmId = 1L; + Long accountId = 2L; + Long zoneId = 3L; + Long offeringId = 4L; + Long backupScheduleId = 5L; + String vmHostName = "vm1"; + String vmUuid = "uuid1"; + String resourceName = "Backup-" + vmHostName + "-" + vmUuid; + + boolean forced = true; + + VMInstanceVO vm = mock(VMInstanceVO.class); + when(vm.getId()).thenReturn(vmId); + when(vm.getDataCenterId()).thenReturn(1L); + when(vm.getBackupOfferingId()).thenReturn(offeringId); + when(vm.getAccountId()).thenReturn(accountId); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getHostName()).thenReturn(vmHostName); + when(vm.getUuid()).thenReturn(vmUuid); + when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + when(vmInstanceDao.update(vmId, vm)).thenReturn(true); + + BackupOfferingVO offering = mock(BackupOfferingVO.class); + when(backupOfferingDao.findById(vm.getBackupOfferingId())).thenReturn(offering); + when(offering.getProvider()).thenReturn("testbackupprovider"); + when(backupProvider.removeVMFromBackupOffering(vm)).thenReturn(true); + when(backupProvider.willDeleteBackupsOnOfferingRemoval()).thenReturn(true); + when(backupDao.listByVmId(null, vmId)).thenReturn(new ArrayList<>()); + + BackupScheduleVO backupSchedule = new BackupScheduleVO(); + ReflectionTestUtils.setField(backupSchedule, "id", backupScheduleId); + when(backupScheduleDao.listByVM(vmId)).thenReturn(List.of(backupSchedule)); + + overrideBackupFrameworkConfigValue(); + + try (MockedStatic usageEventUtilsMocked = Mockito.mockStatic(UsageEventUtils.class)) { + boolean result = backupManager.removeVMFromBackupOffering(vmId, forced); + + assertTrue(result); + verify(vmInstanceDao, times(1)).findByIdIncludingRemoved(vmId); + verify(backupOfferingDao, times(1)).findById(vm.getBackupOfferingId()); + verify(backupManager, times(1)).getBackupProvider("testbackupprovider"); + verify(backupScheduleDao, times(1)).remove(backupScheduleId); + usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVED_AND_BACKUPS_DELETED, accountId, zoneId, vmId, resourceName, + offeringId, null, null, Backup.class.getSimpleName(), vmUuid)); + } + } + + @Test + public void testDeleteBackupScheduleByVmId() { + Long vmId = 1L; + Long scheduleId = 2L; + DeleteBackupScheduleCmd cmd = new DeleteBackupScheduleCmd(); + ReflectionTestUtils.setField(cmd, "vmId", vmId); + + overrideBackupFrameworkConfigValue(); + + VMInstanceVO vm = mock(VMInstanceVO.class); + when(vmInstanceDao.findById(vmId)).thenReturn(vm); + BackupScheduleVO schedule = mock(BackupScheduleVO.class); + when(schedule.getId()).thenReturn(scheduleId); + when(backupScheduleDao.listByVM(vmId)).thenReturn(List.of(schedule)); + when(backupScheduleDao.remove(scheduleId)).thenReturn(true); + + boolean result = backupManager.deleteBackupSchedule(cmd); + assertTrue(result); + } + + @Test + public void testRestoreBackupToVM() throws NoTransitionException { + Long backupId = 1L; + Long vmId = 2L; + Long hostId = 3L; + Long offeringId = 4L; + Long poolId = 5L; + + BackupVO backup = mock(BackupVO.class); + when(backup.getBackupOfferingId()).thenReturn(offeringId); + when(backup.getStatus()).thenReturn(Backup.Status.BackedUp); + + VMInstanceVO vm = mock(VMInstanceVO.class); + when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + when(vm.getId()).thenReturn(vmId); + when(vm.getState()).thenReturn(VirtualMachine.State.Stopped); + when(vm.getHostId()).thenReturn(hostId); + + BackupOfferingVO offering = mock(BackupOfferingVO.class); + BackupProvider backupProvider = mock(BackupProvider.class); + when(backupProvider.supportsInstanceFromBackup()).thenReturn(true); + + overrideBackupFrameworkConfigValue(); + + when(backupDao.findById(backupId)).thenReturn(backup); + when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + when(backupOfferingDao.findByIdIncludingRemoved(offeringId)).thenReturn(offering); + when(offering.getProvider()).thenReturn("testbackupprovider"); + when(backupManager.getBackupProvider("testbackupprovider")).thenReturn(backupProvider); + when(virtualMachineManager.stateTransitTo(vm, VirtualMachine.Event.RestoringRequested, hostId)).thenReturn(true); + when(virtualMachineManager.stateTransitTo(vm, VirtualMachine.Event.RestoringSuccess, hostId)).thenReturn(true); + + VolumeVO rootVolume = mock(VolumeVO.class); + when(rootVolume.getPoolId()).thenReturn(poolId); + HostVO host = mock(HostVO.class); + when(hostDao.findById(hostId)).thenReturn(host); + StoragePoolVO pool = mock(StoragePoolVO.class); + when(volumeDao.findIncludingRemovedByInstanceAndType(vmId, Volume.Type.ROOT)).thenReturn(List.of(rootVolume)); + when(primaryDataStoreDao.findById(poolId)).thenReturn(pool); + when(rootVolume.getPoolId()).thenReturn(poolId); + when(volumeDao.findIncludingRemovedByInstanceAndType(vmId, Volume.Type.ROOT)).thenReturn(List.of(rootVolume)); + when(primaryDataStoreDao.findById(poolId)).thenReturn(pool); + when(backupProvider.restoreBackupToVM(vm, backup, null, null)).thenReturn(true); + + try (MockedStatic utils = Mockito.mockStatic(ActionEventUtils.class)) { + boolean result = backupManager.restoreBackupToVM(backupId, vmId); + + assertTrue(result); + verify(backupProvider, times(1)).restoreBackupToVM(vm, backup, null, null); + verify(virtualMachineManager, times(1)).stateTransitTo(vm, VirtualMachine.Event.RestoringRequested, hostId); + verify(virtualMachineManager, times(1)).stateTransitTo(vm, VirtualMachine.Event.RestoringSuccess, hostId); + } catch (ResourceUnavailableException e) { + fail("Test failed due to exception" + e); + } + } + + @Test + public void testRestoreBackupToVMException() throws NoTransitionException { + Long backupId = 1L; + Long vmId = 2L; + Long hostId = 3L; + Long offeringId = 4L; + Long poolId = 5L; + + BackupVO backup = mock(BackupVO.class); + when(backup.getBackupOfferingId()).thenReturn(offeringId); + when(backup.getStatus()).thenReturn(Backup.Status.BackedUp); + + VMInstanceVO vm = mock(VMInstanceVO.class); + when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + when(vm.getId()).thenReturn(vmId); + when(vm.getState()).thenReturn(VirtualMachine.State.Stopped); + when(vm.getHostId()).thenReturn(hostId); + + BackupOfferingVO offering = mock(BackupOfferingVO.class); + BackupProvider backupProvider = mock(BackupProvider.class); + when(backupProvider.supportsInstanceFromBackup()).thenReturn(true); + + overrideBackupFrameworkConfigValue(); + + when(backupDao.findById(backupId)).thenReturn(backup); + when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + when(backupOfferingDao.findByIdIncludingRemoved(offeringId)).thenReturn(offering); + when(offering.getProvider()).thenReturn("testbackupprovider"); + when(backupManager.getBackupProvider("testbackupprovider")).thenReturn(backupProvider); + when(virtualMachineManager.stateTransitTo(vm, VirtualMachine.Event.RestoringRequested, hostId)).thenReturn(true); + when(virtualMachineManager.stateTransitTo(vm, VirtualMachine.Event.RestoringFailed, hostId)).thenReturn(true); + + VolumeVO rootVolume = mock(VolumeVO.class); + when(rootVolume.getPoolId()).thenReturn(poolId); + HostVO host = mock(HostVO.class); + when(hostDao.findById(hostId)).thenReturn(host); + StoragePoolVO pool = mock(StoragePoolVO.class); + when(volumeDao.findIncludingRemovedByInstanceAndType(vmId, Volume.Type.ROOT)).thenReturn(List.of(rootVolume)); + when(primaryDataStoreDao.findById(poolId)).thenReturn(pool); + when(rootVolume.getPoolId()).thenReturn(poolId); + when(volumeDao.findIncludingRemovedByInstanceAndType(vmId, Volume.Type.ROOT)).thenReturn(List.of(rootVolume)); + when(primaryDataStoreDao.findById(poolId)).thenReturn(pool); + when(backupProvider.restoreBackupToVM(vm, backup, null, null)).thenReturn(false); + + try (MockedStatic utils = Mockito.mockStatic(ActionEventUtils.class)) { + CloudRuntimeException exception = Assert.assertThrows(CloudRuntimeException.class, + () -> backupManager.restoreBackupToVM(backupId, vmId)); + + verify(backupProvider, times(1)).restoreBackupToVM(vm, backup, null, null); + verify(virtualMachineManager, times(1)).stateTransitTo(vm, VirtualMachine.Event.RestoringRequested, hostId); + verify(virtualMachineManager, times(1)).stateTransitTo(vm, VirtualMachine.Event.RestoringFailed, hostId); + } + } + + @Test + public void testGetBackupStorageUsedStats() { + Long zoneId = 1L; + overrideBackupFrameworkConfigValue(); + when(backupManager.getBackupProvider(zoneId)).thenReturn(backupProvider); + when(backupProvider.getBackupStorageStats(zoneId)).thenReturn(new Pair<>(100L, 200L)); + + CapacityVO capacity = backupManager.getBackupStorageUsedStats(zoneId); + + Assert.assertNotNull(capacity); + Assert.assertEquals(Optional.ofNullable(Long.valueOf(100)), Optional.ofNullable(capacity.getUsedCapacity())); + Assert.assertEquals(Optional.ofNullable(Long.valueOf(200)), Optional.ofNullable(capacity.getTotalCapacity())); + Assert.assertEquals(CapacityVO.CAPACITY_TYPE_BACKUP_STORAGE, capacity.getCapacityType()); + } + + @Test + public void testCheckAndRemoveBackupOfferingBeforeExpunge() { + Long vmId = 1L; + Long zoneId = 2L; + Long offeringId = 3L; + String vmUuid = "uuid1"; + String instanceName = "i-2-1-VM"; + String backupExternalId = "backup-external-id"; + + VMInstanceVO vm = mock(VMInstanceVO.class); + when(vm.getId()).thenReturn(vmId); + when(vm.getUuid()).thenReturn(vmUuid); + when(vm.getBackupOfferingId()).thenReturn(offeringId); + when(vm.getInstanceName()).thenReturn(instanceName); + when(vm.getBackupExternalId()).thenReturn(backupExternalId); + when(vm.getDataCenterId()).thenReturn(zoneId); + Backup backup = mock(Backup.class); + when(backupDao.listByVmIdAndOffering(zoneId, vmId, offeringId)).thenReturn(List.of(backup)); + + CloudRuntimeException exception = Assert.assertThrows(CloudRuntimeException.class, + () -> backupManager.checkAndRemoveBackupOfferingBeforeExpunge(vm)); + Assert.assertEquals("This Instance [uuid: uuid1, name: i-2-1-VM] has a " + + "Backup Offering [id: 3, external id: backup-external-id] with 1 backups. Please, remove the backup offering " + + "before proceeding to VM exclusion!", exception.getMessage()); + } + + @Test + public void testGetIpToNetworkMapFromBackup() { + Long networkId1 = 1L; + Long networkId2 = 2L; + String networkUuid1 = "network-uuid-1"; + String networkUuid2 = "network-uuid-2"; + String ip1 = "10.1.1.1"; + String ip2 = "10.1.1.2"; + String ipv61 = "2001:db8::1"; + String ipv62 = "2001:db8::2"; + String mac1 = "00:11:22:33:44:55"; + String mac2 = "00:11:22:33:44:56"; + + // Test case 1: Missing network information + Backup backup1 = mock(Backup.class); + List networkIds1 = new ArrayList<>(); + try { + backupManager.getIpToNetworkMapFromBackup(backup1, true, networkIds1); + fail("Expected CloudRuntimeException for missing network information"); + } catch (CloudRuntimeException e) { + assertEquals("Backup doesn't contain network information. Please specify at least one valid network while creating instance", e.getMessage()); + } + + // Test case 2: IP preservation enabled with IP information + Backup backup2 = mock(Backup.class); + String nicsJson = String.format("[{\"networkid\":\"%s\",\"ipaddress\":\"%s\",\"ip6address\":\"%s\",\"macaddress\":\"%s\"}," + + "{\"networkid\":\"%s\",\"ipaddress\":\"%s\",\"ip6address\":\"%s\",\"macaddress\":\"%s\"}]", + networkUuid1, ip1, ipv61, mac1, networkUuid2, ip2, ipv62, mac2); + when(backup2.getDetail(ApiConstants.NICS)).thenReturn(nicsJson); + + NetworkVO network1 = mock(NetworkVO.class); + NetworkVO network2 = mock(NetworkVO.class); + when(networkDao.findByUuid(networkUuid1)).thenReturn(network1); + when(networkDao.findByUuid(networkUuid2)).thenReturn(network2); + when(network1.getId()).thenReturn(networkId1); + when(network2.getId()).thenReturn(networkId2); + + Network.IpAddresses ipAddresses1 = mock(Network.IpAddresses.class); + Network.IpAddresses ipAddresses2 = mock(Network.IpAddresses.class); + when(networkService.getIpAddressesFromIps(ip1, ipv61, mac1)).thenReturn(ipAddresses1); + when(networkService.getIpAddressesFromIps(ip2, ipv62, mac2)).thenReturn(ipAddresses2); + + List networkIds2 = new ArrayList<>(); + Map result2 = backupManager.getIpToNetworkMapFromBackup(backup2, true, networkIds2); + + assertEquals(2, result2.size()); + assertEquals(ipAddresses1, result2.get(networkId1)); + assertEquals(ipAddresses2, result2.get(networkId2)); + assertEquals(2, networkIds2.size()); + assertTrue(networkIds2.contains(networkId1)); + assertTrue(networkIds2.contains(networkId2)); + + // Test case 3: IP preservation enabled but missing IP information + Backup backup3 = mock(Backup.class); + nicsJson = String.format("[{\"networkid\":\"%s\"}]", networkUuid1); + when(backup3.getDetail(ApiConstants.NICS)).thenReturn(nicsJson); + + List networkIds3 = new ArrayList<>(); + Map result3 = backupManager.getIpToNetworkMapFromBackup(backup3, true, networkIds3); + + assertEquals(1, result3.size()); + assertNull(result3.get(networkId1)); + assertEquals(1, networkIds3.size()); + assertTrue(networkIds3.contains(networkId1)); + + // Test case 4: IP preservation disabled + Backup backup4 = mock(Backup.class); + nicsJson = String.format("[{\"networkid\":\"%s\"}]", networkUuid1); + when(backup4.getDetail(ApiConstants.NICS)).thenReturn(nicsJson); + + List networkIds4 = new ArrayList<>(); + Map result4 = backupManager.getIpToNetworkMapFromBackup(backup4, false, networkIds4); + + assertEquals(1, result4.size()); + assertNull(result4.get(networkId1)); + assertEquals(1, networkIds4.size()); + assertTrue(networkIds4.contains(networkId1)); + } + + @Test + public void testDeleteBackupVmNotFound() { + Long backupId = 1L; + Long vmId = 2L; + Long zoneId = 3L; + Long accountId = 4L; + Long backupOfferingId = 5L; + String vmHostName = "vm1"; + String vmUuid = "uuid1"; + String resourceName = "Backup-" + vmHostName + "-" + vmUuid; + + BackupVO backup = mock(BackupVO.class); + when(backup.getId()).thenReturn(backupId); + when(backup.getVmId()).thenReturn(vmId); + when(backup.getZoneId()).thenReturn(zoneId); + when(backup.getAccountId()).thenReturn(accountId); + when(backup.getBackupOfferingId()).thenReturn(backupOfferingId); + when(backup.getSize()).thenReturn(100L); + + overrideBackupFrameworkConfigValue(); + + VMInstanceVO vm = mock(VMInstanceVO.class); + when(vm.getId()).thenReturn(vmId); + when(vm.getAccountId()).thenReturn(accountId); + when(vm.getBackupOfferingId()).thenReturn(10L); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getHostName()).thenReturn(vmHostName); + when(vm.getUuid()).thenReturn(vmUuid); + when(backupDao.findByIdIncludingRemoved(backupId)).thenReturn(backup); + when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + when(backupDao.listByVmIdAndOffering(zoneId, vmId, backupOfferingId)).thenReturn(new ArrayList<>()); + + BackupOfferingVO offering = mock(BackupOfferingVO.class); + when(backupOfferingDao.findByIdIncludingRemoved(backupOfferingId)).thenReturn(offering); + + when(backupProvider.deleteBackup(backup, false)).thenReturn(true); + + when(backupDao.remove(backupId)).thenReturn(true); + + try (MockedStatic usageEventUtilsMocked = Mockito.mockStatic(UsageEventUtils.class)) { + boolean result = backupManager.deleteBackup(backupId, false); + + assertTrue(result); + verify(backupProvider).deleteBackup(backup, false); + verify(resourceLimitMgr).decrementResourceCount(accountId, Resource.ResourceType.backup); + verify(resourceLimitMgr).decrementResourceCount(accountId, Resource.ResourceType.backup_storage, backup.getSize()); + verify(backupDao).remove(backupId); + usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVED_AND_BACKUPS_DELETED, accountId, zoneId, vmId, resourceName, + backupOfferingId, null, null, Backup.class.getSimpleName(), vmUuid)); + } + } + + @Test + public void testNewBackupResponse() { + Long vmId = 1L; + Long accountId = 2L; + Long domainId = 3L; + Long zoneId = 4L; + Long vmOfferingId = 5L; + Long backupOfferingId = 6L; + Long backupId = 7L; + Long templateId = 8L; + String templateUuid = "template-uuid1"; + String serviceOfferingUuid = "service-offering-uuid1"; + + BackupVO backup = new BackupVO(); + ReflectionTestUtils.setField(backup, "id", backupId); + ReflectionTestUtils.setField(backup, "uuid", "backup-uuid"); + backup.setVmId(vmId); + backup.setAccountId(accountId); + backup.setDomainId(domainId); + backup.setZoneId(zoneId); + backup.setBackupOfferingId(backupOfferingId); + backup.setType("Full"); + backup.setBackupScheduleId(null); + + VMInstanceVO vm = new VMInstanceVO(vmId, 0L, "test-vm", "test-vm", VirtualMachine.Type.User, + 0L, Hypervisor.HypervisorType.Simulator, 0L, domainId, accountId, 0L, false); + vm.setDataCenterId(zoneId); + vm.setBackupOfferingId(vmOfferingId); + vm.setTemplateId(templateId); + + AccountVO account = new AccountVO(); + account.setUuid("account-uuid"); + account.setAccountName("test-account"); + + DomainVO domain = new DomainVO(); + domain.setUuid("domain-uuid"); + domain.setName("test-domain"); + + DataCenterVO zone = new DataCenterVO(1L, "test-zone", null, null, null, null, null, null, null, null, DataCenter.NetworkType.Advanced, null, null); + zone.setUuid("zone-uuid"); + + BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); + Mockito.when(offering.getUuid()).thenReturn("offering-uuid"); + Mockito.when(offering.getName()).thenReturn("test-offering"); + + Mockito.when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + Mockito.when(accountDao.findByIdIncludingRemoved(accountId)).thenReturn(account); + Mockito.when(domainDao.findByIdIncludingRemoved(domainId)).thenReturn(domain); + Mockito.when(dataCenterDao.findByIdIncludingRemoved(zoneId)).thenReturn(zone); + Mockito.when(backupOfferingDao.findByIdIncludingRemoved(backupOfferingId)).thenReturn(offering); + + VMTemplateVO template = mock(VMTemplateVO.class); + when(template.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); + when(template.getUuid()).thenReturn(templateUuid); + when(template.getName()).thenReturn("template1"); + when(vmTemplateDao.findByUuid(templateUuid)).thenReturn(template); + Map details = new HashMap<>(); + details.put(ApiConstants.TEMPLATE_ID, templateUuid); + + ServiceOfferingVO serviceOffering = mock(ServiceOfferingVO.class); + when(serviceOffering.getUuid()).thenReturn(serviceOfferingUuid); + when(serviceOffering.getName()).thenReturn("service-offering1"); + when(serviceOfferingDao.findByUuid(serviceOfferingUuid)).thenReturn(serviceOffering); + details.put(ApiConstants.SERVICE_OFFERING_ID, serviceOfferingUuid); + + NetworkVO network = mock(NetworkVO.class); + when(network.getName()).thenReturn("network1"); + when(networkDao.findByUuid("network-uuid1")).thenReturn(network); + details.put(ApiConstants.NICS, "[{\"networkid\":\"network-uuid1\"}]"); + + Mockito.when(backupDetailsDao.listDetailsKeyPairs(backup.getId(), true)).thenReturn(details); + + BackupResponse response = backupManager.createBackupResponse(backup, true); + + Assert.assertEquals("backup-uuid", response.getId()); + Assert.assertEquals("test-vm", response.getVmName()); + Assert.assertEquals("account-uuid", response.getAccountId()); + Assert.assertEquals("test-account", response.getAccount()); + Assert.assertEquals("domain-uuid", response.getDomainId()); + Assert.assertEquals("test-domain", response.getDomain()); + Assert.assertEquals("zone-uuid", response.getZoneId()); + Assert.assertEquals("test-zone", response.getZone()); + Assert.assertEquals("offering-uuid", response.getBackupOfferingId()); + Assert.assertEquals("test-offering", response.getBackupOffering()); + Assert.assertEquals("MANUAL", response.getIntervalType()); + Assert.assertEquals("{serviceofferingid=service-offering-uuid1, isiso=false, hypervisor=Simulator, " + + "nics=[{\"networkid\":\"network-uuid1\",\"networkname\":\"network1\"}], serviceofferingname=service-offering1, " + + "templatename=template1, templateid=template-uuid1}", response.getVmDetails().toString()); + Assert.assertEquals(true, response.getVmOfferingRemoved()); + } + @Test public void validateAndGetDefaultBackupRetentionIfRequiredTestReturnZeroAsDefaultValue() { int retention = backupManager.validateAndGetDefaultBackupRetentionIfRequired(null, backupOfferingVOMock, null); diff --git a/server/src/test/java/org/apache/cloudstack/resource/ResourceCleanupServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/resource/ResourceCleanupServiceImplTest.java index d214f04ddb4d..c1552b9c4457 100644 --- a/server/src/test/java/org/apache/cloudstack/resource/ResourceCleanupServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/resource/ResourceCleanupServiceImplTest.java @@ -30,6 +30,8 @@ import java.util.concurrent.TimeUnit; import org.apache.cloudstack.api.command.admin.resource.PurgeExpungedResourcesCmd; +import org.apache.cloudstack.backup.BackupVO; +import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.jobs.dao.VmWorkJobDao; @@ -138,12 +140,14 @@ public class ResourceCleanupServiceImplTest { ConsoleSessionDao consoleSessionDao; @Mock ServiceOfferingDetailsDao serviceOfferingDetailsDao; + @Mock + BackupDao backupDao; @Spy @InjectMocks ResourceCleanupServiceImpl resourceCleanupService = Mockito.spy(new ResourceCleanupServiceImpl()); - List ids = List.of(1L, 2L); + List ids = List.of(1L, 2L, 3L); Long batchSize = 100L; private void overrideConfigValue(final ConfigKey configKey, final Object value) { @@ -372,44 +376,53 @@ public void testGetVmIdsWithActiveVolumeSnapshots() { @Test public void testGetFilteredVmIdsForSnapshots() { - Long skippedVmIds = ids.get(0); - Long notSkippedVmIds = ids.get(1); + List skippedVmIds = ids.subList(0, 2); + Long notSkippedVmIds = ids.get(2); VMSnapshotVO vmSnapshotVO = Mockito.mock(VMSnapshotVO.class); Mockito.when(vmSnapshotVO.getVmId()).thenReturn(1L); Mockito.when(vmSnapshotDao.searchByVms(Mockito.anyList())).thenReturn(List.of(vmSnapshotVO)); HashSet set = new HashSet<>(); set.add(1L); Mockito.doReturn(set).when(resourceCleanupService).getVmIdsWithActiveVolumeSnapshots(ids); - Pair, List> result = resourceCleanupService.getFilteredVmIdsForSnapshots(new ArrayList<>(ids)); + BackupVO backupVO = Mockito.mock(BackupVO.class); + Mockito.when(backupVO.getVmId()).thenReturn(2L); + Mockito.when(backupDao.searchByVmIds(Mockito.anyList())).thenReturn(List.of(backupVO)); + Pair, List> result = resourceCleanupService.getFilteredVmIdsForSnapshotsAndBackups(new ArrayList<>(ids)); Assert.assertEquals(1, result.first().size()); - Assert.assertEquals(1, result.second().size()); + Assert.assertEquals(2, result.second().size()); Assert.assertEquals(notSkippedVmIds, result.first().get(0)); - Assert.assertEquals(skippedVmIds, result.second().get(0)); + Assert.assertEquals(skippedVmIds, result.second()); } @Test - public void testGetVmIdsWithNoActiveSnapshots() { + public void testGetVmIdsWithNoActiveSnapshotsAndBackups() { VMInstanceVO vm1 = Mockito.mock(VMInstanceVO.class); Mockito.when(vm1.getId()).thenReturn(ids.get(0)); VMInstanceVO vm2 = Mockito.mock(VMInstanceVO.class); Mockito.when(vm2.getId()).thenReturn(ids.get(1)); + VMInstanceVO vm3 = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm3.getId()).thenReturn(ids.get(2)); Mockito.when(vmInstanceDao.searchRemovedByRemoveDate(Mockito.any(), Mockito.any(), - Mockito.anyLong(), Mockito.anyList())).thenReturn(List.of(vm1, vm2)); - Long skippedVmIds = ids.get(0); - Long notSkippedVmIds = ids.get(1); + Mockito.anyLong(), Mockito.anyList())).thenReturn(List.of(vm1, vm2, vm3)); + List skippedVmIds = ids.subList(0, 2); + Long notSkippedVmIds = ids.get(2); VMSnapshotVO vmSnapshotVO = Mockito.mock(VMSnapshotVO.class); Mockito.when(vmSnapshotVO.getVmId()).thenReturn(1L); Mockito.when(vmSnapshotDao.searchByVms(Mockito.anyList())).thenReturn(List.of(vmSnapshotVO)); HashSet set = new HashSet<>(); set.add(1L); Mockito.doReturn(set).when(resourceCleanupService).getVmIdsWithActiveVolumeSnapshots(Mockito.anyList()); + BackupVO backupVO = Mockito.mock(BackupVO.class); + Mockito.when(backupVO.getVmId()).thenReturn(2L); + Mockito.when(backupDao.searchByVmIds(Mockito.anyList())).thenReturn(List.of(backupVO)); + Pair, List> result = - resourceCleanupService.getVmIdsWithNoActiveSnapshots(new Date(), new Date(), batchSize, + resourceCleanupService.getVmIdsWithNoActiveSnapshotsAndBackups(new Date(), new Date(), batchSize, new ArrayList<>()); Assert.assertEquals(1, result.first().size()); - Assert.assertEquals(1, result.second().size()); + Assert.assertEquals(2, result.second().size()); Assert.assertEquals(notSkippedVmIds, result.first().get(0)); - Assert.assertEquals(skippedVmIds, result.second().get(0)); + Assert.assertEquals(skippedVmIds, result.second()); } @Test @@ -422,7 +435,7 @@ public void testPurgeVMEntitiesNoVms() { @Test public void testPurgeVMEntities() { Mockito.doReturn(new Pair<>(ids, new ArrayList<>())).when(resourceCleanupService) - .getVmIdsWithNoActiveSnapshots(Mockito.any(), Mockito.any(), Mockito.anyLong(), Mockito.anyList()); + .getVmIdsWithNoActiveSnapshotsAndBackups(Mockito.any(), Mockito.any(), Mockito.anyLong(), Mockito.anyList()); Mockito.when(vmInstanceDao.expungeList(ids)).thenReturn(ids.size()); Assert.assertEquals(ids.size(), resourceCleanupService.purgeVMEntities(batchSize, new Date(), new Date())); } @@ -430,14 +443,14 @@ public void testPurgeVMEntities() { @Test public void testExpungeVMEntityFiltered() { Mockito.doReturn(new Pair<>(new ArrayList<>(), List.of(ids.get(0)))).when(resourceCleanupService) - .getFilteredVmIdsForSnapshots(Mockito.anyList()); + .getFilteredVmIdsForSnapshotsAndBackups(Mockito.anyList()); Assert.assertFalse(resourceCleanupService.purgeVMEntity(ids.get(0))); } @Test public void testPurgeVMEntityFiltered() { Mockito.doReturn(new Pair<>(List.of(ids.get(0)), new ArrayList<>())).when(resourceCleanupService) - .getFilteredVmIdsForSnapshots(Mockito.anyList()); + .getFilteredVmIdsForSnapshotsAndBackups(Mockito.anyList()); Mockito.doNothing().when(resourceCleanupService) .purgeLinkedVMEntities(Mockito.anyList(), Mockito.anyLong()); Mockito.when(vmInstanceDao.expunge(ids.get(0))).thenReturn(true); @@ -447,7 +460,7 @@ public void testPurgeVMEntityFiltered() { @Test public void testPurgeVMEntity() { Mockito.doReturn(new Pair<>(List.of(ids.get(0)), new ArrayList<>())).when(resourceCleanupService) - .getFilteredVmIdsForSnapshots(Mockito.anyList()); + .getFilteredVmIdsForSnapshotsAndBackups(Mockito.anyList()); Mockito.doNothing().when(resourceCleanupService) .purgeLinkedVMEntities(Mockito.anyList(), Mockito.anyLong()); Mockito.when(vmInstanceDao.expunge(ids.get(0))).thenReturn(true); diff --git a/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java index 3ce855b504b4..b630ddc69a74 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java @@ -110,6 +110,7 @@ public void testCreateBucket() { ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); + Mockito.when(objectStoreVO.getTotalSize()).thenReturn(2000000000L); Mockito.when(objectStoreDao.findById(poolId)).thenReturn(objectStoreVO); ObjectStoreEntity objectStore = Mockito.mock(ObjectStoreEntity.class); Mockito.when(dataStoreMgr.getDataStore(objectStoreId, DataStoreRole.Object)).thenReturn(objectStore); diff --git a/test/integration/smoke/test_attach_multiple_volumes.py b/test/integration/smoke/test_attach_multiple_volumes.py index 81199bcfdfa1..9357e4d65090 100644 --- a/test/integration/smoke/test_attach_multiple_volumes.py +++ b/test/integration/smoke/test_attach_multiple_volumes.py @@ -285,6 +285,8 @@ def test_attach_multiple_volumes(self): self.query_async_job(vol3_jobId.jobid) self.query_async_job(vol4_jobId.jobid) + time.sleep(60) + # List all the volumes attached to the instance. Includes even the Root disk. list_volume_response = Volume.list( self.apiClient, @@ -337,6 +339,8 @@ def test_attach_and_distribute_multiple_volumes(self): self.query_async_job(vol5_jobId.jobid) self.query_async_job(vol6_jobId.jobid) + time.sleep(60) + volumes = Volume.list(self.apiClient, virtualmachineid=self.virtual_machine.id, type="DATADISK", diff --git a/test/integration/smoke/test_backup_recovery_dummy.py b/test/integration/smoke/test_backup_recovery_dummy.py index b4789bd0f249..3e2f16f8c686 100644 --- a/test/integration/smoke/test_backup_recovery_dummy.py +++ b/test/integration/smoke/test_backup_recovery_dummy.py @@ -18,10 +18,11 @@ from marvin.cloudstackTestCase import cloudstackTestCase from marvin.lib.utils import (cleanup_resources) -from marvin.lib.base import (Account, ServiceOffering, VirtualMachine, BackupOffering, Configurations, Backup) +from marvin.lib.base import (Account, ServiceOffering, DiskOffering, VirtualMachine, BackupOffering, Configurations, Backup, Volume) from marvin.lib.common import (get_domain, get_zone, get_template) from nose.plugins.attrib import attr from marvin.codes import FAILED +import time class TestDummyBackupAndRecovery(cloudstackTestCase): @@ -44,33 +45,34 @@ def setUpClass(cls): cls._cleanup = [] # Check backup configuration values, set them to enable the dummy provider - backup_enabled_cfg = Configurations.list(cls.api_client, name='backup.framework.enabled', zoneid=cls.zone.id) - backup_provider_cfg = Configurations.list(cls.api_client, name='backup.framework.provider.plugin', zoneid=cls.zone.id) + backup_enabled_cfg = Configurations.list(cls.api_client, name='backup.framework.enabled') + backup_provider_cfg = Configurations.list(cls.api_client, name='backup.framework.provider.plugin') cls.backup_enabled = backup_enabled_cfg[0].value cls.backup_provider = backup_provider_cfg[0].value if cls.backup_enabled == "false": - Configurations.update(cls.api_client, 'backup.framework.enabled', value='true', zoneid=cls.zone.id) + cls.skipTest(cls, reason="Test can be run only if the config backup.framework.enabled is true") if cls.backup_provider != "dummy": - Configurations.update(cls.api_client, 'backup.framework.provider.plugin', value='dummy', zoneid=cls.zone.id) + Configurations.update(cls.api_client, 'backup.framework.provider.plugin', value='dummy') if cls.hypervisor.lower() != 'simulator': return cls.account = Account.create(cls.api_client, cls.services["account"], domainid=cls.domain.id) cls.offering = ServiceOffering.create(cls.api_client,cls.services["service_offerings"]["small"]) + cls.diskoffering = DiskOffering.create(cls.api_client, cls.services["disk_offering"]) cls.vm = VirtualMachine.create(cls.api_client, cls.services["small"], accountid=cls.account.name, domainid=cls.account.domainid, serviceofferingid=cls.offering.id, - mode=cls.services["mode"]) - cls._cleanup = [cls.offering, cls.account] + diskofferingid=cls.diskoffering.id, mode=cls.services["mode"]) + cls._cleanup = [cls.offering, cls.diskoffering, cls.account] # Import a dummy backup offering to use on tests cls.provider_offerings = BackupOffering.listExternal(cls.api_client, cls.zone.id) cls.debug("Importing backup offering %s - %s" % (cls.provider_offerings[0].externalid, cls.provider_offerings[0].name)) - cls.offering = BackupOffering.importExisting(cls.api_client, cls.zone.id, cls.provider_offerings[0].externalid, + cls.backup_offering = BackupOffering.importExisting(cls.api_client, cls.zone.id, cls.provider_offerings[0].externalid, cls.provider_offerings[0].name, cls.provider_offerings[0].description) - cls._cleanup.append(cls.offering) + cls._cleanup.append(cls.backup_offering) @classmethod def tearDownClass(cls): @@ -79,10 +81,8 @@ def tearDownClass(cls): cleanup_resources(cls.api_client, cls._cleanup) # Restore original backup framework values values - if cls.backup_enabled == "false": - Configurations.update(cls.api_client, 'backup.framework.enabled', value=cls.backup_enabled, zoneid=cls.zone.id) if cls.backup_provider != "dummy": - Configurations.update(cls.api_client, 'backup.framework.provider.plugin', value=cls.backup_provider, zoneid=cls.zone.id) + Configurations.update(cls.api_client, 'backup.framework.provider.plugin', value=cls.backup_provider) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) @@ -139,7 +139,7 @@ def test_vm_backup_lifecycle(self): self.assertEqual(backups, None, "There should not exist any backup for the VM") # Assign VM to offering and create ad-hoc backup - self.offering.assignOffering(self.apiclient, self.vm.id) + self.backup_offering.assignOffering(self.apiclient, self.vm.id) Backup.create(self.apiclient, self.vm.id) # Verify backup is created for the VM @@ -155,4 +155,68 @@ def test_vm_backup_lifecycle(self): self.assertEqual(backups, None, "There should not exist any backup for the VM") # Remove VM from offering - self.offering.removeOffering(self.apiclient, self.vm.id) + self.backup_offering.removeOffering(self.apiclient, self.vm.id) + + @attr(tags=["advanced", "backup"], required_hardware="false") + def test_vm_backup_create_vm_from_backup(self): + """ + Test creating a new VM from a backup + """ + self.backup_offering.assignOffering(self.apiclient, self.vm.id) + + Backup.create(self.apiclient, self.vm.id, "backup1") + Backup.create(self.apiclient, self.vm.id, "backup2") + + # Verify backup is created for the VM + backups = Backup.list(self.apiclient, self.vm.id) + #self.cleanup.extend(backups) + #self.cleanup.append(backups[0]) + self.assertEqual(len(backups), 2, "There should exist two backups for the VM") + + # Remove VM from offering + self.backup_offering.removeOffering(self.apiclient, self.vm.id) + + # Verify no. of backups after removing the backup offering + backups = Backup.list(self.apiclient, self.vm.id) + self.assertEqual(len(backups), 2, "There should exist two backups for the VM") + + # Create a new VM from first backup + new_vm_name = "vm-from-backup1-" + str(int(time.time())) + new_vm = Backup.createVMFromBackup( + self.apiclient, + self.services["small"], + mode=self.services["mode"], + backupid=backups[0].id, + vmname=new_vm_name, + accountname=self.account.name, + domainid=self.account.domainid, + zoneid=self.zone.id + ) + self.cleanup.append(new_vm) + + # Verify the new VM was created successfully + self.assertIsNotNone(new_vm, "Failed to create VM from backup") + self.assertEqual(new_vm.name, new_vm_name, "VM name does not match the requested name") + + # Verify the new VM is running + self.assertEqual(new_vm.state, "Running", "New VM should be in Running state") + + # Verify the new VM has the correct service offering + self.assertEqual(new_vm.serviceofferingid, self.offering.id, + "New VM should have the correct service offering") + + # Verify the new VM has the correct zone + self.assertEqual(new_vm.zoneid, self.zone.id, "New VM should be in the correct zone") + + # Verify the new VM has the correct number of volumes (ROOT + DATADISK) + volumes = Volume.list( + self.apiclient, + virtualmachineid=new_vm.id, + listall=True + ) + self.assertTrue(isinstance(volumes, list), "List volumes should return a valid list") + self.assertEqual(2, len(volumes), "The new VM should have 2 volumes (ROOT + DATADISK)") + + # Delete backups + Backup.delete(self.apiclient, backups[0].id) + Backup.delete(self.apiclient, backups[1].id) diff --git a/test/integration/smoke/test_backup_recovery_nas.py b/test/integration/smoke/test_backup_recovery_nas.py new file mode 100644 index 000000000000..ea7f1112cbe5 --- /dev/null +++ b/test/integration/smoke/test_backup_recovery_nas.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.utils import (cleanup_resources) +from marvin.lib.base import (Account, ServiceOffering, DiskOffering, VirtualMachine, BackupOffering, + BackupRepository, Backup, Configurations, Volume, StoragePool) +from marvin.lib.common import (get_domain, get_zone, get_template) +from nose.plugins.attrib import attr +from marvin.codes import FAILED +import time + +class TestNASBackupAndRecovery(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + # Setup + + cls.testClient = super(TestNASBackupAndRecovery, cls).getClsTestClient() + cls.api_client = cls.testClient.getApiClient() + cls.services = cls.testClient.getParsedTestDataConfig() + cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests()) + cls.services["mode"] = cls.zone.networktype + cls.hypervisor = cls.testClient.getHypervisorInfo() + cls.domain = get_domain(cls.api_client) + cls.template = get_template(cls.api_client, cls.zone.id, cls.services["ostype"]) + if cls.template == FAILED: + assert False, "get_template() failed to return template with description %s" % cls.services["ostype"] + cls.services["small"]["zoneid"] = cls.zone.id + cls.services["small"]["template"] = cls.template.id + cls._cleanup = [] + + if cls.hypervisor.lower() != 'kvm': + cls.skipTest(cls, reason="Test can be run only on KVM hypervisor") + + cls.storage_pool = StoragePool.list(cls.api_client)[0] + if cls.storage_pool.type.lower() != 'networkfilesystem': + cls.skipTest(cls, reason="Test can be run only if the primary storage is of type NFS") + + # Check backup configuration values, set them to enable the nas provider + backup_enabled_cfg = Configurations.list(cls.api_client, name='backup.framework.enabled') + backup_provider_cfg = Configurations.list(cls.api_client, name='backup.framework.provider.plugin') + cls.backup_enabled = backup_enabled_cfg[0].value + cls.backup_provider = backup_provider_cfg[0].value + + if cls.backup_enabled == "false": + cls.skipTest(cls, reason="Test can be run only if the config backup.framework.enabled is true") + if cls.backup_provider != "nas": + Configurations.update(cls.api_client, 'backup.framework.provider.plugin', value='nas') + + cls.account = Account.create(cls.api_client, cls.services["account"], domainid=cls.domain.id) + + cls._cleanup = [cls.account] + + # Create NAS backup repository and offering. Use the same directory as the storage pool + cls.backup_repository = BackupRepository.add(cls.api_client, zoneid=cls.zone.id, name="Nas", + address=cls.storage_pool.ipaddress + ":" + cls.storage_pool.path, + provider="nas", type="nfs",) + cls._cleanup.append(cls.backup_repository) + cls.provider_offerings = BackupOffering.listExternal(cls.api_client, cls.zone.id) + cls.backup_offering = BackupOffering.importExisting(cls.api_client, cls.zone.id, cls.provider_offerings[0].externalid, + cls.provider_offerings[0].name, cls.provider_offerings[0].description) + cls._cleanup.append(cls.backup_offering) + + cls.offering = ServiceOffering.create(cls.api_client,cls.services["service_offerings"]["small"]) + cls.diskoffering = DiskOffering.create(cls.api_client, cls.services["disk_offering"]) + cls._cleanup.extend([cls.offering, cls.diskoffering, cls.account]) + cls.vm = VirtualMachine.create(cls.api_client, cls.services["small"], accountid=cls.account.name, + domainid=cls.account.domainid, serviceofferingid=cls.offering.id, + diskofferingid=cls.diskoffering.id, mode=cls.services["mode"]) + + + @classmethod + def tearDownClass(cls): + try: + # Cleanup resources used + cleanup_resources(cls.api_client, cls._cleanup) + + if cls.backup_provider != "nas": + Configurations.update(cls.api_client, 'backup.framework.provider.plugin', value=cls.backup_provider) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + if self.hypervisor.lower() != 'kvm': + raise self.skipTest("Skipping test cases which must only run for Simulator") + self.cleanup = [] + + def tearDown(self): + try: + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + @attr(tags=["advanced", "backup"], required_hardware="true") + def test_vm_backup_lifecycle(self): + """ + Test VM backup lifecycle + """ + + # Verify there are no backups for the VM + backups = Backup.list(self.apiclient, self.vm.id) + self.assertEqual(backups, None, "There should not exist any backup for the VM") + + # Assign VM to offering and create ad-hoc backup + self.backup_offering.assignOffering(self.apiclient, self.vm.id) + Backup.create(self.apiclient, self.vm.id) + + # Verify backup is created for the VM + backups = Backup.list(self.apiclient, self.vm.id) + self.assertEqual(len(backups), 1, "There should exist only one backup for the VM") + backup = backups[0] + + # Delete backup + Backup.delete(self.apiclient, backup.id) + + # Verify backup is deleted + backups = Backup.list(self.apiclient, self.vm.id) + self.assertEqual(backups, None, "There should not exist any backup for the VM") + + # Remove VM from offering + self.backup_offering.removeOffering(self.apiclient, self.vm.id) + + @attr(tags=["advanced", "backup"], required_hardware="true") + def test_vm_backup_create_vm_from_backup(self): + """ + Test creating a new VM from a backup + """ + self.backup_offering.assignOffering(self.apiclient, self.vm.id) + + # Create a file and take backup + try: + ssh_client_vm = self.vm.get_ssh_client(reconnect=True) + ssh_client_vm.execute("touch test_backup_and_recovery.txt") + except Exception as err: + self.fail("SSH failed for Virtual machine: %s due to %s" % (self.vm.ipaddress, err)) + + time.sleep(5) + + Backup.create(self.apiclient, self.vm.id, "backup1") + Backup.create(self.apiclient, self.vm.id, "backup2") + + # Verify backup is created for the VM + backups = Backup.list(self.apiclient, self.vm.id) + self.assertEqual(len(backups), 2, "There should exist two backups for the VM") + + # Remove VM from offering + self.backup_offering.removeOffering(self.apiclient, self.vm.id) + + # Verify no. of backups after removing the backup offering + backups = Backup.list(self.apiclient, self.vm.id) + self.assertEqual(len(backups), 2, "There should exist two backups for the VM") + + # Create a new VM from first backup + new_vm_name = "vm-from-backup1-" + str(int(time.time())) + new_vm = Backup.createVMFromBackup( + self.apiclient, + self.services["small"], + mode=self.services["mode"], + backupid=backups[0].id, + vmname=new_vm_name, + accountname=self.account.name, + domainid=self.account.domainid, + zoneid=self.zone.id + ) + self.cleanup.append(new_vm) + + # Verify the new VM was created successfully + self.assertIsNotNone(new_vm, "Failed to create VM from backup") + self.assertEqual(new_vm.name, new_vm_name, "VM name does not match the requested name") + + # Verify the new VM is running + self.assertEqual(new_vm.state, "Running", "New VM should be in Running state") + + # Verify the new VM has the correct service offering + self.assertEqual(new_vm.serviceofferingid, self.offering.id, + "New VM should have the correct service offering") + + # Verify the new VM has the correct zone + self.assertEqual(new_vm.zoneid, self.zone.id, "New VM should be in the correct zone") + + # Verify the new VM has the correct number of volumes (ROOT + DATADISK) + volumes = Volume.list( + self.apiclient, + virtualmachineid=new_vm.id, + listall=True + ) + self.assertTrue(isinstance(volumes, list), "List volumes should return a valid list") + self.assertEqual(2, len(volumes), "The new VM should have 2 volumes (ROOT + DATADISK)") + + # Verify that the file is present in the Instance created from backup + try: + ssh_client_new_vm = new_vm.get_ssh_client(reconnect=True) + result = ssh_client_new_vm.execute("ls test_backup_and_recovery.txt") + self.assertEqual(result[0], "test_backup_and_recovery.txt", + "Instance created from Backup should have the same file as the backup.") + except Exception as err: + self.fail("SSH failed for Virtual machine: %s due to %s" % (self.vm.ipaddress, err)) + + # Delete backups + Backup.delete(self.apiclient, backups[0].id) + Backup.delete(self.apiclient, backups[1].id) diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index 0c9843f43993..ebfc4816015d 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -516,6 +516,43 @@ def access_ssh_over_nat( virtual_machine.ssh_ip = nat_rule.ipaddress virtual_machine.public_ip = nat_rule.ipaddress + @classmethod + def program_ssh_access( + cls, apiclient, services, mode, networkids, virtual_machine, allow_egress=False, vpcid=None): + """ + Program SSH access to the VM + """ + # program ssh access over NAT via PF + retries = 5 + interval = 30 + while retries > 0: + try: + if mode.lower() == 'advanced': + cls.access_ssh_over_nat( + apiclient, + services, + virtual_machine, + allow_egress=allow_egress, + networkid=networkids[0] if networkids else None, + vpcid=vpcid) + elif mode.lower() == 'basic': + if virtual_machine.publicip is not None: + # EIP/ELB (netscaler) enabled zone + vm_ssh_ip = virtual_machine.publicip + else: + # regular basic zone with security group + vm_ssh_ip = virtual_machine.nic[0].ipaddress + virtual_machine.ssh_ip = vm_ssh_ip + virtual_machine.public_ip = vm_ssh_ip + break + except Exception as e: + if retries >= 0: + retries = retries - 1 + time.sleep(interval) + continue + raise Exception( + "The following exception appeared while programming ssh access - %s" % e) + @classmethod def create(cls, apiclient, services, templateid=None, accountid=None, domainid=None, zoneid=None, networkids=None, @@ -716,36 +753,7 @@ def create(cls, apiclient, services, templateid=None, accountid=None, virtual_machine.public_ip = virtual_machine.nic[0].ipaddress return VirtualMachine(virtual_machine.__dict__, services) - # program ssh access over NAT via PF - retries = 5 - interval = 30 - while retries > 0: - time.sleep(interval) - try: - if mode.lower() == 'advanced': - cls.access_ssh_over_nat( - apiclient, - services, - virtual_machine, - allow_egress=allow_egress, - networkid=cmd.networkids[0] if cmd.networkids else None, - vpcid=vpcid) - elif mode.lower() == 'basic': - if virtual_machine.publicip is not None: - # EIP/ELB (netscaler) enabled zone - vm_ssh_ip = virtual_machine.publicip - else: - # regular basic zone with security group - vm_ssh_ip = virtual_machine.nic[0].ipaddress - virtual_machine.ssh_ip = vm_ssh_ip - virtual_machine.public_ip = vm_ssh_ip - break - except Exception as e: - if retries >= 0: - retries = retries - 1 - continue - raise Exception( - "The following exception appeared while programming ssh access - %s" % e) + cls.program_ssh_access(apiclient, services, mode, cmd.networkids, virtual_machine, allow_egress, vpcid) return VirtualMachine(virtual_machine.__dict__, services) @@ -6157,11 +6165,13 @@ def __init__(self, items): self.__dict__.update(items) @classmethod - def create(self, apiclient, vmid): + def create(cls, apiclient, vmid, name=None): """Create VM backup""" cmd = createBackup.createBackupCmd() cmd.virtualmachineid = vmid + if name: + cmd.name = name return Backup(apiclient.createBackup(cmd).__dict__) @classmethod @@ -6175,11 +6185,12 @@ def delete(self, apiclient, id, forced=None): return (apiclient.deleteBackup(cmd)) @classmethod - def list(self, apiclient, vmid): + def list(self, apiclient, vmid=None): """List VM backups""" cmd = listBackups.listBackupsCmd() - cmd.virtualmachineid = vmid + if vmid: + cmd.virtualmachineid = vmid cmd.listall = True return (apiclient.listBackups(cmd)) @@ -6201,6 +6212,21 @@ def restoreVolumeFromBackupAndAttachToVM(self, apiclient, backupid, volumeid, vi cmd.virtualmachineid = virtualmachineid return (apiclient.restoreVolumeFromBackupAndAttachToVM(cmd)) + @classmethod + def createVMFromBackup(cls, apiclient, services, mode, backupid, accountname, domainid, zoneid, vmname=None): + """Create new VM from backup + """ + cmd = createVMFromBackup.createVMFromBackupCmd() + cmd.backupid = backupid + cmd.account = accountname + cmd.domainid = domainid + cmd.zoneid = zoneid + if vmname: + cmd.name = vmname + response = apiclient.createVMFromBackup(cmd) + virtual_machine = VirtualMachine(response.__dict__, []) + VirtualMachine.program_ssh_access(apiclient, services, mode, cmd.networkids, virtual_machine) + return virtual_machine class BackupSchedule: @@ -6243,6 +6269,37 @@ def update(self, apiclient, vmid, **kwargs): return (apiclient.updateBackupSchedule(cmd)) +class BackupRepository: + + def __init__(self, items): + self.__dict__.update(items) + + @classmethod + def add(cls, apiclient, zoneid, name, address, provider, type): + """Add backup repository""" + + cmd = addBackupRepository.addBackupRepositoryCmd() + cmd.zoneid = zoneid + cmd.name = name + cmd.address = address + cmd.provider = provider + cmd.type = type + response = apiclient.addBackupRepository(cmd) + return BackupRepository(response.__dict__) + + def delete(self, apiclient): + """Delete backup repository""" + + cmd = deleteBackupRepository.deleteBackupRepositoryCmd() + cmd.id = self.id + return (apiclient.deleteBackupRepository(cmd)) + + def list(self, apiclient): + """List backup repository""" + + cmd = listBackupRepositories.listBackupRepositoriesCmd() + return (apiclient.listBackupRepository(cmd)) + class ProjectRole: def __init__(self, items): diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index d3a447894894..29b8166d8877 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -581,6 +581,7 @@ "label.configuration": "Configuration", "label.configuration.details": "Configuration Details", "label.configure": "Configure", + "label.configure.instance": "Configure Instance", "label.configure.health.monitor": "Configure Health Monitor", "label.configure.app": "Configure the App", "label.configure.ldap": "Configure LDAP", @@ -1043,6 +1044,7 @@ "label.f5.ip.loadbalancer": "F5 BIG-IP load balancer.", "label.failed": "Failed", "label.featured": "Featured", +"label.fetch.from.backup": "Fetch from Backup", "label.fetch.instances": "Fetch Instances", "label.fetch.latest": "Fetch latest", "label.filename": "File Name", @@ -1243,6 +1245,7 @@ "label.instance": "Instance", "label.instance.conversion.support": "Instance Conversion Supported", "label.instance.groups": "Instance Groups", +"label.instance.metadata": "Instance metadata", "label.instance.name": "Instance name", "label.instancename": "Internal name", "label.instanceport": "Instance port", @@ -1590,6 +1593,7 @@ "label.more.access.dashboard.ui": "More about accessing dashboard UI", "label.mount.cks.iso.on.vr": "Use CKS packages from Virtual Router", "label.mount.sharedfs": "Mount Shared FileSystem via NFS", +"label.mountopts": "Mount options", "label.move.down.row": "Move down one row", "label.move.to.bottom": "Move to bottom", "label.move.to.top": "Move to top", @@ -2083,6 +2087,8 @@ "label.restartrequired": "Restart required", "label.restore": "Restore", "label.restore.volume.attach": "Restore volume and attach", +"label.use.backup.ip.address": "Use IP Addresses from Backup", +"label.use.backup.ip.address.tooltip": "Use the same IP/MAC addresses as stored in the backup metadata. The command will error out if the IP/MAC addresses are not available", "label.review": "Review", "label.role": "Role", "label.roleid": "Role", @@ -2291,6 +2297,9 @@ "label.srx": "SRX", "label.srx.firewall": "Juniper SRX firewall", "label.storageaccessgroups": "Storage Access Groups", +"label.storageallocated": "Allocated size", +"label.storagetotal": "Total size", +"label.storageused": "Used size", "label.clusterstorageaccessgroups": "Cluster Storage Access Groups", "label.podstorageaccessgroups": "Pod Storage Access Groups", "label.zonestorageaccessgroups": "Zone Storage Access Groups", @@ -2823,6 +2832,7 @@ "label.cniconfiguration": "CNI Configuration", "label.cniconfigname": "Associated CNI Configuration", "label.cniconfigparams": "CNI Configuration parameters", +"label.create.instance.from.backup": "Create new instance from backup", "label.lease.enable": "Enable Lease", "label.lease.enable.tooltip": "The Instance Lease feature allows to set a lease duration (in days) for instances, after which they automatically expire. Upon expiry, the instance can either be stopped (powered off) or destroyed, based on the configured policy", "label.instance.lease": "Instance lease", @@ -2837,6 +2847,7 @@ "message.action.cancel.maintenance": "Your host has been successfully canceled for maintenance. This process can take up to several minutes.", "message.action.cancel.maintenance.mode": "Please confirm that you want to cancel this maintenance.", "message.action.create.snapshot.from.vmsnapshot": "Please confirm that you want to create Snapshot from Instance Snapshot", +"message.action.create.instance.from.backup": "Please confirm that you want to create a new Instance from the given Backup.
Click on configure to edit the parameters for the new Instance before creation.", "message.action.delete.asnrange": "Please confirm the AS range that you want to delete", "message.action.delete.autoscale.vmgroup": "Please confirm that you want to delete this autoscaling group.", "message.action.delete.backup.offering": "Please confirm that you want to delete this backup offering?", @@ -3041,7 +3052,7 @@ "message.autoscale.vm.networks": "Please choose at least one Network for Instances in the autoscaling group. The default Network must be an Isolated Network or VPC Network Tier which supports Instance AutoScaling and has load balancing rules.", "message.autoscale.vmprofile.update": "The autoscale Instance profile can be updated only when autoscaling group is DISABLED.", "message.backup.attach.restore": "Please confirm that you want to restore and attach the volume from the backup?", -"message.backup.create": "Are you sure you want create an Instance backup?", +"message.backup.create": "Are you sure you want to create an Instance backup?", "message.backup.offering.remove": "Are you sure you want to remove Instance from backup offering and delete the backup chain?", "message.backup.restore": "Please confirm that you want to restore the Instance backup?", "message.cancel.shutdown": "Please confirm that you would like to cancel the shutdown on this Management Server. It will resume accepting any new Async Jobs.", @@ -3138,12 +3149,14 @@ "message.confirm.unmanage.gpu.devices": "Please confirm that you want to unmanage the selected GPU devices?", "message.confirm.upgrade.router.newer.template": "Please confirm that you want to upgrade router to use newer Template.", "message.cpu.usage.info": "The CPU usage percentage can exceed 100% if the Instance has more than 1 vCPU or when CPU Cap is not enabled. This behavior happens according to the hypervisor being used (e.g: in KVM), due to how they account the stats", +"message.create.backup.failed": "Failed to create backup.", "message.create.bucket.failed": "Failed to create bucket.", "message.create.bucket.processing": "Bucket creation in progress", "message.create.compute.offering": "Compute Offering created", "message.create.sharedfs.failed": "Failed to create Shared FileSystem.", "message.create.sharedfs.processing": "Shared FileSystem creation in progress.", "message.create.tungsten.public.network": "Create Tungsten-Fabric public Network", +"message.create.instance.from.backup.prefill": "Data is prefilled using the configurations stored in the backup. Edit to change individual fields.", "message.create.internallb": "Creating internal LB", "message.create.internallb.failed": "Failed to create internal LB.", "message.create.internallb.processing": "Creation of internal LB is in progress", diff --git a/ui/src/components/view/BackupMetadata.vue b/ui/src/components/view/BackupMetadata.vue new file mode 100644 index 000000000000..fea93a4e9c72 --- /dev/null +++ b/ui/src/components/view/BackupMetadata.vue @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + diff --git a/ui/src/components/view/DeployVMFromBackup.vue b/ui/src/components/view/DeployVMFromBackup.vue new file mode 100644 index 000000000000..8d929a1fed0f --- /dev/null +++ b/ui/src/components/view/DeployVMFromBackup.vue @@ -0,0 +1,2663 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + + + diff --git a/ui/src/components/view/DetailsTab.vue b/ui/src/components/view/DetailsTab.vue index 3a4d0fa043b6..24c2043b90a8 100644 --- a/ui/src/components/view/DetailsTab.vue +++ b/ui/src/components/view/DetailsTab.vue @@ -64,7 +64,8 @@
- {{ volume.type }} - {{ volume.path }} ({{ parseFloat(volume.size / (1024.0 * 1024.0 * 1024.0)).toFixed(1) }} GB) + {{ volume.type }} - {{ volume.path }} + {{ volume.type }} - {{ volume.path }} ({{ parseFloat(volume.size / (1024.0 * 1024.0 * 1024.0)).toFixed(1) }} GB)
diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 205e340652d7..85e416cbbcf0 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -342,7 +342,8 @@ }} + @@ -110,7 +116,7 @@ export default { }, computed: { columns () { - return [ + const cols = [ { key: 'icon', title: '', @@ -134,7 +140,17 @@ export default { key: 'keep', title: this.$t('label.keep'), dataIndex: 'maxbackups' - }, + } + ] + const hasQuiesce = this.dataSource.some(item => 'quiescevm' in item) + if (hasQuiesce) { + cols.push({ + key: 'quiescevm', + title: this.$t('label.quiescevm'), + dataIndex: 'quiescevm' + }) + } + cols.push( { key: 'timezone', title: this.$t('label.timezone'), @@ -146,7 +162,8 @@ export default { dataIndex: 'actions', width: 80 } - ] + ) + return cols } }, mounted () { diff --git a/ui/src/views/compute/backup/FormSchedule.vue b/ui/src/views/compute/backup/FormSchedule.vue index 01cae9d7d8f4..643ae116916d 100644 --- a/ui/src/views/compute/backup/FormSchedule.vue +++ b/ui/src/views/compute/backup/FormSchedule.vue @@ -132,6 +132,14 @@ + + + + + +
import { ref, reactive, toRaw } from 'vue' -import { postAPI } from '@/api' +import { getAPI, postAPI } from '@/api' import { timeZone } from '@/utils/timezone' import { mixinForm } from '@/utils/mixin' import debounce from 'lodash/debounce' +import TooltipLabel from '@/components/widgets/TooltipLabel' export default { name: 'FormSchedule', mixins: [mixinForm], + components: { + TooltipLabel + }, props: { loading: { type: Boolean, @@ -185,13 +197,18 @@ export default { dayOfMonth: [], timeZoneMap: [], fetching: false, + backupProvider: null, actionLoading: false, listDayOfWeek: ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'] } }, + beforeCreate () { + this.apiParams = this.$getApiParams('createBackupSchedule') + }, created () { this.initForm() this.fetchTimeZone() + this.fetchBackupOffering() }, inject: ['refreshSchedule', 'closeSchedule'], methods: { @@ -208,6 +225,16 @@ export default { timezone: [{ required: true, message: `${this.$t('message.error.select')}` }] }) }, + fetchBackupOffering () { + getAPI('listBackupOfferings', { id: this.resource.backupofferingid }).then(json => { + if (json.listbackupofferingsresponse && json.listbackupofferingsresponse.backupoffering) { + const backupoffering = json.listbackupofferingsresponse.backupoffering[0] + this.backupProvider = backupoffering.provider + } + }).catch(error => { + this.$notifyError(error) + }) + }, fetchTimeZone (value) { this.timeZoneMap = [] this.fetching = true @@ -261,6 +288,9 @@ export default { params.intervaltype = values.intervaltype params.maxbackups = values.maxbackups params.timezone = values.timezone + if (values.quiescevm) { + params.quiescevm = values.quiescevm + } switch (values.intervaltype) { case 'hourly': params.schedule = values.time diff --git a/ui/src/views/compute/wizard/NetworkConfiguration.vue b/ui/src/views/compute/wizard/NetworkConfiguration.vue index b2f05dbb6a52..f3192d278bdc 100644 --- a/ui/src/views/compute/wizard/NetworkConfiguration.vue +++ b/ui/src/views/compute/wizard/NetworkConfiguration.vue @@ -77,6 +77,14 @@ +
+ + {{ $t('label.fetch.from.backup') }} + + + {{ $t('label.clear') }} + +
@@ -219,28 +227,31 @@ export default { }, updateNetworkData (name, key, value) { this.formRef.value.validate().then(() => { - this.$emit('handler-error', false) - const index = this.networks.findIndex(item => item.key === key) - if (index === -1) { - const networkItem = {} - networkItem.key = key - networkItem[name] = value - this.networks.push(networkItem) - this.$emit('update-network-config', this.networks) - return - } - - this.networks.filter((item, index) => { - if (item.key === key) { - this.networks[index][name] = value - } - }) + this.updateNetworkDataWithoutValidation(name, key, value) this.$emit('update-network-config', this.networks) }).catch((error) => { this.formRef.value.scrollToField(error.errorFields[0].name) this.$emit('handler-error', true) }) }, + updateNetworkDataWithoutValidation (name, key, value) { + this.$emit('handler-error', false) + const index = this.networks.findIndex(item => item.key === key) + if (index === -1) { + const networkItem = {} + networkItem.key = key + networkItem[name] = value + this.networks.push(networkItem) + this.$emit('update-network-config', this.networks) + return + } + + this.networks.filter((item, index) => { + if (item.key === key) { + this.networks[index][name] = value + } + }) + }, removeItem (id) { this.dataItems = this.dataItems.filter(item => item.id !== id) if (this.selectedRowKeys.includes(id)) { @@ -250,6 +261,59 @@ export default { } } }, + handleFetchIpAddresses () { + if (!this.preFillContent.networkids) { + return + } + if (!this.preFillContent.ipAddresses && !this.preFillContent.macAddresses) { + return + } + + const networkIds = this.dataItems.map(item => item.id) + this.dataItems.forEach(record => { + const ipAddressKey = 'ipAddress' + record.id + const macAddressKey = 'macAddress' + record.id + this.form[ipAddressKey] = '' + this.form[macAddressKey] = '' + }) + + networkIds.forEach((networkId) => { + const backupIndex = this.preFillContent.networkids.findIndex(id => id === networkId) + if (backupIndex !== -1) { + if (this.preFillContent.ipAddresses && backupIndex < this.preFillContent.ipAddresses.length) { + const ipAddress = this.preFillContent.ipAddresses[backupIndex] + if (ipAddress) { + const ipAddressKey = 'ipAddress' + networkId + this.form[ipAddressKey] = ipAddress + this.updateNetworkDataWithoutValidation('ipAddress', networkId, ipAddress) + } + } + + if (this.preFillContent.macAddresses && backupIndex < this.preFillContent.macAddresses.length) { + const macAddress = this.preFillContent.macAddresses[backupIndex] + if (macAddress) { + const macAddressKey = 'macAddress' + networkId + this.form[macAddressKey] = macAddress + this.updateNetworkDataWithoutValidation('macAddress', networkId, macAddress) + } + } + } + }) + }, + handleClearIpAddresses () { + this.dataItems.forEach(record => { + const ipAddressKey = 'ipAddress' + record.id + const macAddressKey = 'macAddress' + record.id + this.form[ipAddressKey] = '' + this.form[macAddressKey] = '' + + this.updateNetworkDataWithoutValidation('ipAddress', record.id, '') + this.updateNetworkDataWithoutValidation('macAddress', record.id, '') + }) + + this.networks = [] + this.$emit('update-network-config', this.networks) + }, async validatorMacAddress (rule, value) { if (!value || value === '') { return Promise.resolve() diff --git a/ui/src/views/compute/wizard/NetworkSelection.vue b/ui/src/views/compute/wizard/NetworkSelection.vue index 3a4fd8c88f90..fffcbf7e3e75 100644 --- a/ui/src/views/compute/wizard/NetworkSelection.vue +++ b/ui/src/views/compute/wizard/NetworkSelection.vue @@ -260,8 +260,11 @@ export default { }) if (!this.loading) { if (this.preFillContent.networkids) { - this.selectedRowKeys = this.preFillContent.networkids - this.$emit('select-network-item', this.preFillContent.networkids) + const validNetworkIds = this.preFillContent.networkids.filter(networkId => + this.items.some(item => item.id === networkId) + ) + this.selectedRowKeys = validNetworkIds + this.$emit('select-network-item', validNetworkIds) } else { if (this.items && this.items.length > 0) { if (this.oldZoneId === this.zoneId) { diff --git a/ui/src/views/compute/wizard/TemplateIsoSelection.vue b/ui/src/views/compute/wizard/TemplateIsoSelection.vue index 9393a7860de3..4979068dac70 100644 --- a/ui/src/views/compute/wizard/TemplateIsoSelection.vue +++ b/ui/src/views/compute/wizard/TemplateIsoSelection.vue @@ -25,7 +25,7 @@ @@ -103,12 +103,19 @@ export default { deep: true, handler (items) { const key = this.inputDecorator.slice(0, -2) + if (this.pagination) { + return + } for (const filter of this.filterOpts) { - if (items[filter.id] && items[filter.id][key] && items[filter.id][key].length > 0) { - if (!this.pagination) { + if (this.preFillContent.templateid) { + if (items[filter.id]?.[key]?.some(item => item.id === this.preFillContent.templateid)) { this.filterType = filter.id this.checkedValue = items[filter.id][key][0].id + break } + } else if (items[filter.id]?.[key]?.length > 0) { + this.filterType = filter.id + this.checkedValue = items[filter.id][key][0].id break } } diff --git a/ui/src/views/compute/wizard/VolumeDiskOfferingSelectView.vue b/ui/src/views/compute/wizard/VolumeDiskOfferingSelectView.vue new file mode 100644 index 000000000000..4fc7c3fc9727 --- /dev/null +++ b/ui/src/views/compute/wizard/VolumeDiskOfferingSelectView.vue @@ -0,0 +1,280 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/views/dashboard/CapacityDashboard.vue b/ui/src/views/dashboard/CapacityDashboard.vue index 7e0b8180ac84..9b3746288863 100644 --- a/ui/src/views/dashboard/CapacityDashboard.vue +++ b/ui/src/views/dashboard/CapacityDashboard.vue @@ -210,14 +210,14 @@ - +
-
+
{{ $t(ts[ctype]) }} @@ -377,6 +377,8 @@ export default { MEMORY: 'label.memory', PRIVATE_IP: 'label.management.ips', SECONDARY_STORAGE: 'label.secondary.storage', + BACKUP_STORAGE: 'label.backup.storage', + OBJECT_STORAGE: 'label.object.storage', STORAGE: 'label.primary.storage.used', STORAGE_ALLOCATED: 'label.primary.storage.allocated', VIRTUAL_NETWORK_PUBLIC_IP: 'label.public.ips', @@ -438,6 +440,8 @@ export default { case 'STORAGE': case 'STORAGE_ALLOCATED': case 'SECONDARY_STORAGE': + case 'BACKUP_STORAGE': + case 'OBJECT_STORAGE': case 'LOCAL_STORAGE': value = parseFloat(value / (1024 * 1024 * 1024.0), 10).toFixed(2) if (value >= 1024.0) { @@ -667,6 +671,13 @@ export default { min-height: 370px; } +.dashboard-storage { + width: 100%; + overflow-x:hidden; + overflow-y: scroll; + max-height: 370px; +} + .dashboard-event { width: 100%; overflow-x:hidden; diff --git a/ui/src/views/infra/AddObjectStorage.vue b/ui/src/views/infra/AddObjectStorage.vue index dca3b719b6c7..5410a9b9502f 100644 --- a/ui/src/views/infra/AddObjectStorage.vue +++ b/ui/src/views/infra/AddObjectStorage.vue @@ -25,10 +25,16 @@ layout="vertical" @finish="handleSubmit" > - + + - + + - + + - + + + + +
-
{{ $t('label.cancel') }} {{ $t('label.ok') }} @@ -95,6 +109,7 @@ import { ref, reactive, toRaw } from 'vue' import { getAPI } from '@/api' import { mixinForm } from '@/utils/mixin' import ResourceIcon from '@/components/view/ResourceIcon' +import TooltipLabel from '@/components/widgets/TooltipLabel' export default { name: 'AddObjectStorage', @@ -106,7 +121,8 @@ export default { } }, components: { - ResourceIcon + ResourceIcon, + TooltipLabel }, inject: ['parentFetchData'], data () { @@ -116,6 +132,9 @@ export default { loading: false } }, + beforeCreate () { + this.apiParams = this.$getApiParams('addObjectStoragePool') + }, created () { this.initForm() this.fetchData() @@ -147,7 +166,8 @@ export default { const values = this.handleRemoveFields(formRaw) var data = { - name: values.name + name: values.name, + size: values.size } var provider = values.provider diff --git a/ui/src/views/storage/CreateVMFromBackup.vue b/ui/src/views/storage/CreateVMFromBackup.vue new file mode 100644 index 000000000000..f9dbf535d060 --- /dev/null +++ b/ui/src/views/storage/CreateVMFromBackup.vue @@ -0,0 +1,263 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 2e97e238f098..49d799997162 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -1075,7 +1075,7 @@ private boolean isVmSnapshotOnPrimaryEvent(String eventType) { private boolean isBackupEvent(String eventType) { return eventType != null && ( eventType.equals(EventTypes.EVENT_VM_BACKUP_OFFERING_ASSIGN) || - eventType.equals(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE) || + eventType.equals(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVED_AND_BACKUPS_DELETED) || eventType.equals(EventTypes.EVENT_VM_BACKUP_USAGE_METRIC)); } @@ -2030,10 +2030,10 @@ private void createBackupEvent(final UsageEventVO event) { if (EventTypes.EVENT_VM_BACKUP_OFFERING_ASSIGN.equals(event.getType())) { final UsageBackupVO backupVO = new UsageBackupVO(zoneId, accountId, domainId, vmId, backupOfferingId, created); usageBackupDao.persist(backupVO); - } else if (EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE.equals(event.getType())) { - usageBackupDao.removeUsage(accountId, vmId, event.getCreateDate()); + } else if (EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVED_AND_BACKUPS_DELETED.equals(event.getType())) { + usageBackupDao.removeUsage(accountId, vmId, backupOfferingId, event.getCreateDate()); } else if (EventTypes.EVENT_VM_BACKUP_USAGE_METRIC.equals(event.getType())) { - usageBackupDao.updateMetrics(vmId, event.getSize(), event.getVirtualSize()); + usageBackupDao.updateMetrics(vmId, backupOfferingId, event.getSize(), event.getVirtualSize()); } }