diff --git a/conf/db/upgrade/V4.5.3__schema.sql b/conf/db/upgrade/V4.5.3__schema.sql index e7cc58776e7..e11f64cc5bc 100644 --- a/conf/db/upgrade/V4.5.3__schema.sql +++ b/conf/db/upgrade/V4.5.3__schema.sql @@ -1,3 +1,73 @@ alter table LicenseHistoryVO modify COLUMN `userName` varchar(64) NOT NULL; ALTER TABLE LicenseHistoryVO ADD COLUMN capacity int(10) NOT NULL; + +CREATE TABLE IF NOT EXISTS `zstack`.`IAM2VirtualIDInformationVO` ( + `uuid` varchar(32) NOT NULL UNIQUE, + `phone` varchar(255), + `mail` varchar(255), + PRIMARY KEY (`uuid`), + CONSTRAINT `fkIAM2VirtualIDInformationVOIAM2VirtualIDVO` FOREIGN KEY (`uuid`) REFERENCES `IAM2VirtualIDVO` (`uuid`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DELIMITER $$ +CREATE PROCEDURE attributePhoneToInformation() + BEGIN + DECLARE vitualIdPhone VARCHAR(32); + DECLARE vitualIdUuid VARCHAR(32); + DECLARE done INT DEFAULT FALSE; + DECLARE phoneCursor CURSOR FOR SELECT virtualIDUuid, value from `zstack`.`IAM2VirtualIDAttributeVO` WHERE name = "phone"; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE; + + open phoneCursor; + read_loop: LOOP + FETCH phoneCursor INTO vitualIdUuid, vitualIdPhone; + IF done THEN + LEAVE read_loop; + END IF; + + IF (select count(*) from IAM2VirtualIDInformationVO where uuid = vitualIdUuid) = 0 THEN + INSERT `zstack`.`IAM2VirtualIDInformationVO`(uuid, phone) values (vitualIdUuid, vitualIdPhone); + else + update `zstack`.`IAM2VirtualIDInformationVO` set phone = vitualIdPhone where uuid = vitualIdUuid; + END IF; + + END LOOP; + close phoneCursor; + SELECT CURTIME(); + END $$ +DELIMITER ; + +call attributePhoneToInformation(); +DROP PROCEDURE IF EXISTS attributePhoneToInformation; + +DELIMITER $$ +CREATE PROCEDURE attributeMailToInformation() + BEGIN + DECLARE vitualIdMail VARCHAR(32); + DECLARE vitualIdUuid VARCHAR(32); + DECLARE done INT DEFAULT FALSE; + DECLARE mailCursor CURSOR FOR SELECT virtualIDUuid, value from `zstack`.`IAM2VirtualIDAttributeVO` WHERE name = "mail"; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE; + + open mailCursor; + read_loop: LOOP + FETCH mailCursor INTO vitualIdUuid, vitualIdMail; + IF done THEN + LEAVE read_loop; + END IF; + + IF (select count(*) from IAM2VirtualIDInformationVO where uuid = vitualIdUuid) = 0 THEN + INSERT `zstack`.`IAM2VirtualIDInformationVO`(uuid, mail) values (vitualIdUuid, vitualIdMail); + ELSE + update `zstack`.`IAM2VirtualIDInformationVO` set mail = vitualIdMail where uuid = vitualIdUuid; + END IF; + + END LOOP; + close mailCursor; + SELECT CURTIME(); + END $$ +DELIMITER ; + +call attributeMailToInformation(); +DROP PROCEDURE IF EXISTS attributeMailToInformation; diff --git a/core/src/main/java/org/zstack/core/encrypt/EncryptFacadeImpl.java b/core/src/main/java/org/zstack/core/encrypt/EncryptFacadeImpl.java index 5aee3c032f8..a2439bcaa6d 100644 --- a/core/src/main/java/org/zstack/core/encrypt/EncryptFacadeImpl.java +++ b/core/src/main/java/org/zstack/core/encrypt/EncryptFacadeImpl.java @@ -1,12 +1,11 @@ package org.zstack.core.encrypt; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; import org.zstack.core.Platform; import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.componentloader.PluginRegistry; -import org.zstack.core.config.GlobalConfig; -import org.zstack.core.config.GlobalConfigBeforeUpdateExtensionPoint; -import org.zstack.core.config.GlobalConfigUpdateExtensionPoint; +import org.zstack.core.config.*; import org.zstack.core.convert.PasswordConverter; import org.zstack.core.db.DatabaseFacade; import org.zstack.core.db.SQLBatch; @@ -273,6 +272,123 @@ public void updateGlobalConfig(GlobalConfig oldConfig, GlobalConfig newConfig) { } } }); + } + + protected void handleNewAddedEncryptEntity() { + if (PasswordEncryptType.None.toString().equals(EncryptGlobalConfig.ENABLE_PASSWORD_ENCRYPT.value())) { + return; + } + + List metadataVOList = Q.New(EncryptEntityMetadataVO.class) + .eq(EncryptEntityMetadataVO_.state, EncryptEntityState.NewAdded) + .list(); + + new SQLBatch() { + @Override + protected void scripts() { + for (EncryptEntityMetadataVO metadata : metadataVOList) { + // do encrypt + long count = SQL.New(String.format("select count(1) from %s", metadata.getEntityName()), Long.class).find(); + metadata.setState(EncryptEntityState.Encrypting); + metadata = dbf.updateAndRefresh(metadata); + String className = metadata.getEntityName(); + String fieldName = metadata.getColumnName(); + sql(String.format("select uuid from %s", metadata.getEntityName()), String.class) + .limit(1000) + .paginate(count, (List uuids) -> { + for (String uuid : uuids) { + String value = sql(String.format("select %s from %s where uuid = '%s'", fieldName, className, uuid)).find(); + + try { + // If part of the data has been encrypted, first decrypt all the data before encrypting + String decryptedString = decrypt(value); + String encryptedString = encrypt(decryptedString); + + String sql = String.format("update %s set %s = :encrypted where uuid = :uuid", className, fieldName); + + Query query = dbf.getEntityManager().createQuery(sql); + query.setParameter("encrypted", encryptedString); + query.setParameter("uuid", uuid); + query.executeUpdate(); + } catch (Exception e) { + logger.debug(String.format("encrypt error because : %s", e.getMessage())); + } + } + + }); + metadata.setState(EncryptEntityState.Encrypted); + dbf.updateAndRefresh(metadata); + } + } + }.execute(); + } + + private void collectEncryptEntityMetadata() { + for (Field field : encryptedFields) { + List classNames = new ArrayList<>(); + + if (field.getDeclaringClass().getAnnotation(Entity.class) != null && field.getDeclaringClass().getAnnotation(Table.class) != null) { + classNames.add(field.getDeclaringClass().getSimpleName()); + } else { + classNames.addAll(BeanUtils.reflections.getSubTypesOf(field.getDeclaringClass()).stream() + .filter(aClass -> aClass.getAnnotation(Entity.class) != null && aClass.getAnnotation(Table.class) != null) + .map(Class::getSimpleName) + .collect(Collectors.toList())); + } + + for (String className : classNames) { + createIfNotExists(className, field.getName()); + } + } + } + + private void createIfNotExists(String entity, String column) { + if (Q.New(EncryptEntityMetadataVO.class) + .eq(EncryptEntityMetadataVO_.entityName, entity) + .eq(EncryptEntityMetadataVO_.columnName, column) + .isExists()) { + return; + } + + EncryptEntityMetadataVO metadataVO = new EncryptEntityMetadataVO(); + metadataVO.setColumnName(column); + metadataVO.setEntityName(entity); + metadataVO.setState(EncryptEntityState.NewAdded); + dbf.persist(metadataVO); + } + + public void updateEncryptDataStateIfExists(String entity, String column, EncryptEntityState state) { + String sql = String.format("update EncryptEntityMetadataVO set state = :state where columnName = :columnName and entityName = :entityName"); + Query query = dbf.getEntityManager().createQuery(sql); + query.setParameter("state", state); + query.setParameter("entityName", entity); + query.setParameter("columnName", column); + query.executeUpdate(); + } + + @Transactional + public void removeConvertRecoverData() { + if (Q.New(EncryptEntityMetadataVO.class) + .isExists()) { + return; + } + + if (PasswordEncryptType.None.toString().equals(EncryptGlobalConfig.ENABLE_PASSWORD_ENCRYPT.value())) { + return; + } + + decryptAllPassword(); + encryptAllPassword(); + } + + @Override + public boolean start() { + initEncryptDriver(); + collectAllEncryptPassword(); + installGlobalConfigUpdateHooks(); + removeConvertRecoverData(); + collectEncryptEntityMetadata(); + handleNewAddedEncryptEntity(); return true; } diff --git a/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostPasswordEncryptCase.groovy b/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostPasswordEncryptCase.groovy index 5eefa304ca3..d5e06d6a629 100644 --- a/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostPasswordEncryptCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/kvm/host/HostPasswordEncryptCase.groovy @@ -74,6 +74,69 @@ class HostPasswordEncryptCase extends SubCase { assert password == "password" } + BeanUtils.reflections.getFieldsAnnotatedWith(Convert.class).forEach({ ec -> + if (ec.getDeclaringClass().getAnnotation(Entity.class) == null || ec.getDeclaringClass().getAnnotation(Table.class) == null) { + return; + } + if (!ec.getAnnotation(Convert.class).converter().equals(PasswordConverter.class)) { + return; + } + + String entityName = ec.getDeclaringClass().getSimpleName(); + String columnName = ec.getName(); + retryInSecs { + assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, entityName).eq(EncryptEntityMetadataVO_.columnName, columnName).findValue() == EncryptEntityState.Encrypted + } + }) + + assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()).findValue() == EncryptEntityState.Encrypted + + // test for unencrypted strings Decryption failures + SQL.New(EncryptEntityMetadataVO.class) + .eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()) + .set(EncryptEntityMetadataVO_.state, EncryptEntityState.NewAdded) + .update() + + ((EncryptFacadeImpl) encryptFacade).handleNewAddedEncryptEntity() + + assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()).findValue() == EncryptEntityState.Encrypted + + retryInSecs { + password = Q.New(KVMHostVO.class).select(KVMHostVO_.password).eq(KVMHostVO_.uuid, host.uuid).findValue() + + assert password == encryptFacade.encrypt("password") + } + + // test handleNewAddedEncryptEntity again, result unchanged + SQL.New(EncryptEntityMetadataVO.class) + .eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()) + .set(EncryptEntityMetadataVO_.state, EncryptEntityState.NewAdded) + .update() + + ((EncryptFacadeImpl) encryptFacade).handleNewAddedEncryptEntity() + + assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()).findValue() == EncryptEntityState.Encrypted + + retryInSecs { + password = Q.New(KVMHostVO.class).select(KVMHostVO_.password).eq(KVMHostVO_.uuid, host.uuid).findValue() + + assert password == encryptFacade.encrypt("password") + } + + List needToChangeHosts = Q.New(KVMHostVO.class).select(KVMHostVO_.password, KVMHostVO_.uuid).listTuple() + needToChangeHosts.forEach({ changeHost -> + SQL.New(KVMHostVO.class) + .eq(KVMHostVO_.uuid, changeHost.get(1, String.class)) + .set(KVMHostVO_.password, encryptFacade.decrypt(changeHost.get(0, String.class))) + .update() + }) + + retryInSecs { + password = Q.New(KVMHostVO.class).select(KVMHostVO_.password).eq(KVMHostVO_.uuid, host.uuid).findValue() + + assert password == "password" + } + updateGlobalConfig { category = EncryptGlobalConfig.CATEGORY name = EncryptGlobalConfig.ENABLE_PASSWORD_ENCRYPT.name @@ -85,6 +148,32 @@ class HostPasswordEncryptCase extends SubCase { assert password == "password" } + + BeanUtils.reflections.getFieldsAnnotatedWith(Convert.class).forEach({ ec -> + if (ec.getDeclaringClass().getAnnotation(Entity.class) == null || ec.getDeclaringClass().getAnnotation(Table.class) == null) { + return; + } + if (!ec.getAnnotation(Convert.class).converter().equals(PasswordConverter.class)) { + return; + } + + String entityName = ec.getDeclaringClass().getSimpleName(); + String columnName = ec.getName(); + assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, entityName).eq(EncryptEntityMetadataVO_.columnName, columnName).findValue() == EncryptEntityState.NewAdded + + }) + + assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()).findValue() == EncryptEntityState.NewAdded + + ((EncryptFacadeImpl) encryptFacade).handleNewAddedEncryptEntity() + + assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()).findValue() == EncryptEntityState.NewAdded + + retryInSecs { + password = Q.New(KVMHostVO.class).select(KVMHostVO_.password).eq(KVMHostVO_.uuid, host.uuid).findValue() + + assert password == "password" + } }