Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

<fix>[encrypt]: recover iam2 attarbute data #1328

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

<fix>[encrypt]: recover iam2 attarbute data #1328

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions conf/db/upgrade/V4.5.3__schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,73 @@
alter table LicenseHistoryVO modify COLUMN `userName` varchar(64) NOT NULL;

ALTER TABLE LicenseHistoryVO ADD COLUMN capacity int(10) NOT NULL;

CREATE TABLE IF NOT EXISTS `zstack`.`IAM2VirtualIDInformationVO` (
`uuid` varchar(32) NOT NULL UNIQUE,
`phone` varchar(255),
`mail` varchar(255),
PRIMARY KEY (`uuid`),
CONSTRAINT `fkIAM2VirtualIDInformationVOIAM2VirtualIDVO` FOREIGN KEY (`uuid`) REFERENCES `IAM2VirtualIDVO` (`uuid`) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

DELIMITER $$
CREATE PROCEDURE attributePhoneToInformation()
BEGIN
DECLARE vitualIdPhone VARCHAR(32);
DECLARE vitualIdUuid VARCHAR(32);
DECLARE done INT DEFAULT FALSE;
DECLARE phoneCursor CURSOR FOR SELECT virtualIDUuid, value from `zstack`.`IAM2VirtualIDAttributeVO` WHERE name = "phone";
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;

open phoneCursor;
read_loop: LOOP
FETCH phoneCursor INTO vitualIdUuid, vitualIdPhone;
IF done THEN
LEAVE read_loop;
END IF;

IF (select count(*) from IAM2VirtualIDInformationVO where uuid = vitualIdUuid) = 0 THEN
INSERT `zstack`.`IAM2VirtualIDInformationVO`(uuid, phone) values (vitualIdUuid, vitualIdPhone);
else
update `zstack`.`IAM2VirtualIDInformationVO` set phone = vitualIdPhone where uuid = vitualIdUuid;
END IF;

END LOOP;
close phoneCursor;
SELECT CURTIME();
END $$
DELIMITER ;

call attributePhoneToInformation();
DROP PROCEDURE IF EXISTS attributePhoneToInformation;

DELIMITER $$
CREATE PROCEDURE attributeMailToInformation()
BEGIN
DECLARE vitualIdMail VARCHAR(32);
DECLARE vitualIdUuid VARCHAR(32);
DECLARE done INT DEFAULT FALSE;
DECLARE mailCursor CURSOR FOR SELECT virtualIDUuid, value from `zstack`.`IAM2VirtualIDAttributeVO` WHERE name = "mail";
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;

open mailCursor;
read_loop: LOOP
FETCH mailCursor INTO vitualIdUuid, vitualIdMail;
IF done THEN
LEAVE read_loop;
END IF;

IF (select count(*) from IAM2VirtualIDInformationVO where uuid = vitualIdUuid) = 0 THEN
INSERT `zstack`.`IAM2VirtualIDInformationVO`(uuid, mail) values (vitualIdUuid, vitualIdMail);
ELSE
update `zstack`.`IAM2VirtualIDInformationVO` set mail = vitualIdMail where uuid = vitualIdUuid;
END IF;

END LOOP;
close mailCursor;
SELECT CURTIME();
END $$
DELIMITER ;

call attributeMailToInformation();
DROP PROCEDURE IF EXISTS attributeMailToInformation;
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

with a brief code review:

  1. The patch will modify the table "LicenseHistoryVO" by adding a new column "capacity" which is of type int(10) and not null.
  2. It will also create a new table "IAM2VirtualIDInformationVO" with columns "uuid", "phone" and "mail". The "uuid" column is the primary key and have a foreign key reference to the "IAM2VirtualIDVO" table.
  3. Create two stored procedures called "attributePhoneToInformation" and "attributeMailToInformation". These procedures will loop through the "IAM2VirtualIDAttributeVO" table and populate the newly created "IAM2VirtualIDInformationVO" table with values from the "value" column.

In terms of potential bugs, I would recommend checking that all the data types match in the foreign key reference. Additionally, make sure that the "uuid" column is properly indexed for both tables.

Finally, it may be worth considering adding a timestamp column to the "IAM2VirtualIDInformationVO" table to keep track of when the rows were inserted or updated.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the review:

  1. The code should be checked to make sure that it adheres to the database's best practices and coding standards.
  2. The foreign key should be checked to make sure that it is correctly referencing the primary key of the other table.
  3. The cursor should be checked to make sure it is properly handling errors and that all data is being handled safely.
  4. The stored procedures should be checked to ensure that they are properly constructed and that they are not introducing any potential security risks.
  5. The code should also be checked for optimization opportunities and for any unnecessary or redundant code.

122 changes: 119 additions & 3 deletions core/src/main/java/org/zstack/core/encrypt/EncryptFacadeImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
package org.zstack.core.encrypt;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
import org.zstack.core.Platform;
import org.zstack.core.cloudbus.CloudBus;
import org.zstack.core.componentloader.PluginRegistry;
import org.zstack.core.config.GlobalConfig;
import org.zstack.core.config.GlobalConfigBeforeUpdateExtensionPoint;
import org.zstack.core.config.GlobalConfigUpdateExtensionPoint;
import org.zstack.core.config.*;
import org.zstack.core.convert.PasswordConverter;
import org.zstack.core.db.DatabaseFacade;
import org.zstack.core.db.SQLBatch;
Comment on lines 3 to 11
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

使用通配符导入 (org.zstack.core.config.*) 可能会导致命名空间冲突，并且降低了代码的可读性。建议明确列出所需的类。

Expand Down Expand Up @@ -273,6 +272,123 @@ public void updateGlobalConfig(GlobalConfig oldConfig, GlobalConfig newConfig) {
}
}
});
}

protected void handleNewAddedEncryptEntity() {
if (PasswordEncryptType.None.toString().equals(EncryptGlobalConfig.ENABLE_PASSWORD_ENCRYPT.value())) {
return;
}

List<EncryptEntityMetadataVO> metadataVOList = Q.New(EncryptEntityMetadataVO.class)
.eq(EncryptEntityMetadataVO_.state, EncryptEntityState.NewAdded)
.list();

new SQLBatch() {
@Override
protected void scripts() {
for (EncryptEntityMetadataVO metadata : metadataVOList) {
// do encrypt
long count = SQL.New(String.format("select count(1) from %s", metadata.getEntityName()), Long.class).find();
metadata.setState(EncryptEntityState.Encrypting);
metadata = dbf.updateAndRefresh(metadata);
String className = metadata.getEntityName();
String fieldName = metadata.getColumnName();
sql(String.format("select uuid from %s", metadata.getEntityName()), String.class)
.limit(1000)
.paginate(count, (List<String> uuids) -> {
for (String uuid : uuids) {
String value = sql(String.format("select %s from %s where uuid = '%s'", fieldName, className, uuid)).find();

try {
// If part of the data has been encrypted, first decrypt all the data before encrypting
String decryptedString = decrypt(value);
String encryptedString = encrypt(decryptedString);

String sql = String.format("update %s set %s = :encrypted where uuid = :uuid", className, fieldName);

Query query = dbf.getEntityManager().createQuery(sql);
query.setParameter("encrypted", encryptedString);
query.setParameter("uuid", uuid);
query.executeUpdate();
} catch (Exception e) {
logger.debug(String.format("encrypt error because : %s", e.getMessage()));
}
}

});
metadata.setState(EncryptEntityState.Encrypted);
dbf.updateAndRefresh(metadata);
}
}
}.execute();
}

private void collectEncryptEntityMetadata() {
for (Field field : encryptedFields) {
List<String> classNames = new ArrayList<>();

if (field.getDeclaringClass().getAnnotation(Entity.class) != null && field.getDeclaringClass().getAnnotation(Table.class) != null) {
classNames.add(field.getDeclaringClass().getSimpleName());
} else {
classNames.addAll(BeanUtils.reflections.getSubTypesOf(field.getDeclaringClass()).stream()
.filter(aClass -> aClass.getAnnotation(Entity.class) != null && aClass.getAnnotation(Table.class) != null)
.map(Class::getSimpleName)
.collect(Collectors.toList()));
}

for (String className : classNames) {
createIfNotExists(className, field.getName());
}
}
}

private void createIfNotExists(String entity, String column) {
if (Q.New(EncryptEntityMetadataVO.class)
.eq(EncryptEntityMetadataVO_.entityName, entity)
.eq(EncryptEntityMetadataVO_.columnName, column)
.isExists()) {
return;
}

EncryptEntityMetadataVO metadataVO = new EncryptEntityMetadataVO();
metadataVO.setColumnName(column);
metadataVO.setEntityName(entity);
metadataVO.setState(EncryptEntityState.NewAdded);
dbf.persist(metadataVO);
}

public void updateEncryptDataStateIfExists(String entity, String column, EncryptEntityState state) {
String sql = String.format("update EncryptEntityMetadataVO set state = :state where columnName = :columnName and entityName = :entityName");
Query query = dbf.getEntityManager().createQuery(sql);
query.setParameter("state", state);
query.setParameter("entityName", entity);
query.setParameter("columnName", column);
query.executeUpdate();
}

@Transactional
public void removeConvertRecoverData() {
if (Q.New(EncryptEntityMetadataVO.class)
.isExists()) {
return;
}

if (PasswordEncryptType.None.toString().equals(EncryptGlobalConfig.ENABLE_PASSWORD_ENCRYPT.value())) {
return;
}

decryptAllPassword();
encryptAllPassword();
}

@Override
public boolean start() {
initEncryptDriver();
collectAllEncryptPassword();
installGlobalConfigUpdateHooks();
removeConvertRecoverData();
collectEncryptEntityMetadata();
handleNewAddedEncryptEntity();

return true;
}
Expand Down
89 changes: 89 additions & 0 deletions test/src/test/groovy/org/zstack/test/integration/kvm/host/HostPasswordEncryptCase.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,69 @@ class HostPasswordEncryptCase extends SubCase {
assert password == "password"
}

BeanUtils.reflections.getFieldsAnnotatedWith(Convert.class).forEach({ ec ->
if (ec.getDeclaringClass().getAnnotation(Entity.class) == null || ec.getDeclaringClass().getAnnotation(Table.class) == null) {
return;
}
if (!ec.getAnnotation(Convert.class).converter().equals(PasswordConverter.class)) {
return;
}

String entityName = ec.getDeclaringClass().getSimpleName();
String columnName = ec.getName();
retryInSecs {
assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, entityName).eq(EncryptEntityMetadataVO_.columnName, columnName).findValue() == EncryptEntityState.Encrypted
}
})

assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()).findValue() == EncryptEntityState.Encrypted

// test for unencrypted strings Decryption failures
SQL.New(EncryptEntityMetadataVO.class)
.eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName())
.set(EncryptEntityMetadataVO_.state, EncryptEntityState.NewAdded)
.update()

((EncryptFacadeImpl) encryptFacade).handleNewAddedEncryptEntity()

assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()).findValue() == EncryptEntityState.Encrypted

retryInSecs {
password = Q.New(KVMHostVO.class).select(KVMHostVO_.password).eq(KVMHostVO_.uuid, host.uuid).findValue()

assert password == encryptFacade.encrypt("password")
}

// test handleNewAddedEncryptEntity again, result unchanged
SQL.New(EncryptEntityMetadataVO.class)
.eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName())
.set(EncryptEntityMetadataVO_.state, EncryptEntityState.NewAdded)
.update()

((EncryptFacadeImpl) encryptFacade).handleNewAddedEncryptEntity()

assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()).findValue() == EncryptEntityState.Encrypted

retryInSecs {
password = Q.New(KVMHostVO.class).select(KVMHostVO_.password).eq(KVMHostVO_.uuid, host.uuid).findValue()

assert password == encryptFacade.encrypt("password")
}

List<Tuple> needToChangeHosts = Q.New(KVMHostVO.class).select(KVMHostVO_.password, KVMHostVO_.uuid).listTuple()
needToChangeHosts.forEach({ changeHost ->
SQL.New(KVMHostVO.class)
.eq(KVMHostVO_.uuid, changeHost.get(1, String.class))
.set(KVMHostVO_.password, encryptFacade.decrypt(changeHost.get(0, String.class)))
.update()
})

retryInSecs {
password = Q.New(KVMHostVO.class).select(KVMHostVO_.password).eq(KVMHostVO_.uuid, host.uuid).findValue()

assert password == "password"
}

updateGlobalConfig {
category = EncryptGlobalConfig.CATEGORY
name = EncryptGlobalConfig.ENABLE_PASSWORD_ENCRYPT.name
Expand All @@ -85,6 +148,32 @@ class HostPasswordEncryptCase extends SubCase {

assert password == "password"
}

BeanUtils.reflections.getFieldsAnnotatedWith(Convert.class).forEach({ ec ->
if (ec.getDeclaringClass().getAnnotation(Entity.class) == null || ec.getDeclaringClass().getAnnotation(Table.class) == null) {
return;
}
if (!ec.getAnnotation(Convert.class).converter().equals(PasswordConverter.class)) {
return;
}

String entityName = ec.getDeclaringClass().getSimpleName();
String columnName = ec.getName();
assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, entityName).eq(EncryptEntityMetadataVO_.columnName, columnName).findValue() == EncryptEntityState.NewAdded

})

assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()).findValue() == EncryptEntityState.NewAdded

((EncryptFacadeImpl) encryptFacade).handleNewAddedEncryptEntity()

assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()).findValue() == EncryptEntityState.NewAdded

retryInSecs {
password = Q.New(KVMHostVO.class).select(KVMHostVO_.password).eq(KVMHostVO_.uuid, host.uuid).findValue()

assert password == "password"
}
}


Expand Down