diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java index 601bdf0ea72..b25941ca676 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java @@ -172,14 +172,13 @@ public int incrementDelegationTokenSeqNum() { */ private OzoneSecretKey updateCurrentKey(KeyPair keyPair, X509Certificate certificate) { - logger.info("Updating current master key for generating tokens. Cert id {}", - certificate.getSerialNumber().toString()); - int newCurrentId = incrementCurrentKeyId(); OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId, certificate.getNotAfter().getTime(), keyPair, certificate.getSerialNumber().toString()); currentKey.set(newKey); + logger.info("Updated current master key for generating tokens. Cert id {}, Master key id {}", + certificate.getSerialNumber().toString(), newKey.getKeyId()); return newKey; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/SecretKeyManagerService.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/SecretKeyManagerService.java index 6b77350cc8c..50c7401dbb0 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/SecretKeyManagerService.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/SecretKeyManagerService.java @@ -158,7 +158,6 @@ public void stop() { } public static boolean isSecretKeyEnable(SecurityConfig conf) { - return conf.isSecurityEnabled() && - (conf.isBlockTokenEnabled() || conf.isContainerTokenEnabled()); + return conf.isSecurityEnabled(); } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java index 19f3e7c4a25..d4db2689612 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java @@ -23,12 +23,14 @@ import java.io.IOException; import java.time.Instant; import java.util.Arrays; +import java.util.UUID; +import com.google.common.base.Preconditions; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.io.DataInputBuffer; -import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMTokenProto; @@ -47,7 +49,11 @@ public class OzoneTokenIdentifier extends AbstractDelegationTokenIdentifier { public static final Text KIND_NAME = new Text("OzoneToken"); + @Deprecated + // the certificate id of this OM, deprecated since HDDS-8829 private String omCertSerialId; + // shared secret key id generated by SCM. + private String secretKeyId; private Type tokenType; private String awsAccessId; private String signature; @@ -82,31 +88,6 @@ public Text getKind() { return KIND_NAME; } - /** Instead of relying on proto serialization, this - * provides explicit serialization for OzoneTokenIdentifier. - * @return byte[] - */ - public byte[] toUniqueSerializedKey() { - DataOutputBuffer buf = new DataOutputBuffer(); - try { - super.write(buf); - WritableUtils.writeVInt(buf, getTokenType().getNumber()); - // Set s3 specific fields. - if (getTokenType().equals(S3AUTHINFO)) { - WritableUtils.writeString(buf, getAwsAccessId()); - WritableUtils.writeString(buf, getSignature()); - WritableUtils.writeString(buf, getStrToSign()); - } else { - WritableUtils.writeString(buf, getOmCertSerialId()); - WritableUtils.writeString(buf, getOmServiceId()); - } - } catch (java.io.IOException e) { - throw new IllegalArgumentException( - "Can't encode the the raw data ", e); - } - return buf.getData(); - } - /** Instead of relying on proto deserialization, this * provides explicit deserialization for OzoneTokenIdentifier. * @return byte[] @@ -125,20 +106,19 @@ public OzoneTokenIdentifier fromUniqueSerializedKey(byte[] rawData) setStrToSign(WritableUtils.readString(in)); } else { this.tokenType = Type.DELEGATION_TOKEN; - setOmCertSerialId(WritableUtils.readString(in)); + String value = WritableUtils.readString(in); + try { + UUID.fromString(value); + setSecretKeyId(value); + } catch (IllegalArgumentException e) { + setOmCertSerialId(value); + } setOmServiceId(WritableUtils.readString(in)); } return this; } - /** - * Overrides default implementation to write using Protobuf. - * - * @param out output stream - * @throws IOException - */ - @Override - public void write(DataOutput out) throws IOException { + public OMTokenProto toProtoBuf() throws IOException { OMTokenProto.Builder builder = OMTokenProto.newBuilder() .setMaxDate(getMaxDate()) .setType(getTokenType()) @@ -155,14 +135,28 @@ public void write(DataOutput out) throws IOException { .setSignature(getSignature()) .setStrToSign(getStrToSign()); } else { - builder.setOmCertSerialId(getOmCertSerialId()); + if (StringUtils.isNotEmpty(getOmCertSerialId())) { + builder.setOmCertSerialId(getOmCertSerialId()); + } + if (StringUtils.isNotEmpty(getSecretKeyId())) { + builder.setSecretKeyId(getSecretKeyId()); + } if (getOmServiceId() != null) { builder.setOmServiceId(getOmServiceId()); } } + return builder.build(); + } - OMTokenProto token = builder.build(); - out.write(token.toByteArray()); + /** + * Overrides default implementation to write using Protobuf. + * + * @param out output stream + * @throws IOException + */ + @Override + public void write(DataOutput out) throws IOException { + out.write(toProtoBuf().toByteArray()); } /** @@ -183,7 +177,12 @@ public void readFields(DataInput in) throws IOException { setMaxDate(token.getMaxDate()); setSequenceNumber(token.getSequenceNumber()); setMasterKeyId(token.getMasterKeyId()); - setOmCertSerialId(token.getOmCertSerialId()); + if (token.hasOmCertSerialId()) { + setOmCertSerialId(token.getOmCertSerialId()); + } + if (token.hasSecretKeyId()) { + setSecretKeyId(token.getSecretKeyId()); + } // Set s3 specific fields. if (getTokenType().equals(S3AUTHINFO)) { @@ -221,7 +220,12 @@ public static OzoneTokenIdentifier readProtoBuf(DataInput in) identifier.setSequenceNumber(token.getSequenceNumber()); identifier.setMasterKeyId(token.getMasterKeyId()); } - identifier.setOmCertSerialId(token.getOmCertSerialId()); + if (token.hasOmCertSerialId()) { + identifier.setOmCertSerialId(token.getOmCertSerialId()); + } + if (token.hasSecretKeyId()) { + identifier.setSecretKeyId(token.getSecretKeyId()); + } identifier.setOmServiceId(token.getOmServiceId()); return identifier; } @@ -264,6 +268,7 @@ public boolean equals(Object obj) { } OzoneTokenIdentifier that = (OzoneTokenIdentifier) obj; return new EqualsBuilder() + .append(getSecretKeyId(), that.getSecretKeyId()) .append(getOmCertSerialId(), that.getOmCertSerialId()) .append(getMaxDate(), that.getMaxDate()) .append(getIssueDate(), that.getIssueDate()) @@ -326,6 +331,18 @@ public String getOmCertSerialId() { public void setOmCertSerialId(String omCertSerialId) { this.omCertSerialId = omCertSerialId; + Preconditions.checkArgument(this.omCertSerialId == null || this.secretKeyId == null, + "omCertSerialId and secretKeyId cannot both be valid"); + } + + public String getSecretKeyId() { + return secretKeyId; + } + + public void setSecretKeyId(String id) { + this.secretKeyId = id; + Preconditions.checkArgument(this.omCertSerialId == null || this.secretKeyId == null, + "omCertSerialId and secretKeyId cannot both be valid"); } public String getOmServiceId() { @@ -383,7 +400,8 @@ public String toString() { .append(", signature=").append(getSignature()) .append(", awsAccessKeyId=").append(getAwsAccessId()) .append(", omServiceId=").append(getOmServiceId()) - .append(", omCertSerialId=").append(getOmCertSerialId()); + .append(", omCertSerialId=").append(getOmCertSerialId()) + .append(", secretKeyId=").append(getSecretKeyId()); return buffer.toString(); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeysApi.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeysApi.java index fb92d91ee71..45458182344 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeysApi.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeysApi.java @@ -23,7 +23,6 @@ import org.apache.hadoop.hdds.protocol.SecretKeyProtocol; import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; -import org.apache.hadoop.hdds.security.exception.SCMSecretKeyException; import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.minikdc.MiniKdc; @@ -67,7 +66,6 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY; import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY; -import static org.apache.hadoop.hdds.security.exception.SCMSecretKeyException.ErrorCode.SECRET_KEY_NOT_ENABLED; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getSecretKeyClientForDatanode; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; @@ -245,24 +243,14 @@ public void testSecretKeyApiSuccess() throws Exception { } /** - * Verify API behavior when block token is not enable. + * Verify API behavior. */ @Test - public void testSecretKeyApiNotEnabled() throws Exception { + public void testSecretKeyApi() throws Exception { startCluster(1); SecretKeyProtocol secretKeyProtocol = getSecretKeyProtocol(); - - SCMSecretKeyException ex = assertThrows(SCMSecretKeyException.class, - secretKeyProtocol::getCurrentSecretKey); - assertEquals(SECRET_KEY_NOT_ENABLED, ex.getErrorCode()); - - ex = assertThrows(SCMSecretKeyException.class, - () -> secretKeyProtocol.getSecretKey(UUID.randomUUID())); - assertEquals(SECRET_KEY_NOT_ENABLED, ex.getErrorCode()); - - ex = assertThrows(SCMSecretKeyException.class, - secretKeyProtocol::getAllSecretKeys); - assertEquals(SECRET_KEY_NOT_ENABLED, ex.getErrorCode()); + assertNull(secretKeyProtocol.getSecretKey(UUID.randomUUID())); + assertEquals(1, secretKeyProtocol.getAllSecretKeys().size()); } /** diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java index 4f41d516153..637e8bd9e4f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java @@ -64,6 +64,7 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.security.exception.SCMSecurityException; import org.apache.hadoop.hdds.security.SecurityConfig; +import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; import org.apache.hadoop.hdds.security.symmetric.SecretKeyManager; import org.apache.hadoop.hdds.security.x509.certificate.authority.CAType; import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultApprover; @@ -100,6 +101,7 @@ import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB; import org.apache.hadoop.ozone.security.OMCertificateClient; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; +import org.apache.hadoop.ozone.security.SecretKeyTestClient; import org.apache.hadoop.security.KerberosAuthException; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; import org.apache.hadoop.security.SecurityUtil; @@ -152,7 +154,6 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.USER_MISMATCH; import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; -import org.apache.ozone.test.LambdaTestUtils; import org.apache.ozone.test.tag.Flaky; import org.apache.ozone.test.tag.Unhealthy; import org.apache.ratis.protocol.ClientId; @@ -1182,10 +1183,10 @@ public String renewAndStoreKeyAndCertificate(boolean force) throws CertificateEx } /** - * Tests delegation token renewal after a certificate renew. + * Tests delegation token renewal after a secret key rotation. */ @Test - void testDelegationTokenRenewCrossCertificateRenew() throws Exception { + void testDelegationTokenRenewCrossSecretKeyRotation() throws Exception { initSCM(); try { scm = HddsTestUtils.getScmSimple(conf); @@ -1206,11 +1207,12 @@ void testDelegationTokenRenewCrossCertificateRenew() throws Exception { CertificateClientTestImpl certClient = new CertificateClientTestImpl(newConf, true); - X509Certificate omCert = certClient.getCertificate(); - String omCertId1 = omCert.getSerialNumber().toString(); // Start OM om.setCertClient(certClient); om.setScmTopologyClient(new ScmTopologyClient(scmBlockClient)); + SecretKeyTestClient secretKeyClient = new SecretKeyTestClient(); + ManagedSecretKey secretKey1 = secretKeyClient.getCurrentSecretKey(); + om.setSecretKeyClient(secretKeyClient); om.start(); GenericTestUtils.waitFor(() -> om.isLeaderReady(), 100, 10000); @@ -1231,30 +1233,26 @@ void testDelegationTokenRenewCrossCertificateRenew() throws Exception { assertEquals(SecurityUtil.buildTokenService( om.getNodeDetails().getRpcAddress()).toString(), token1.getService().toString()); - assertEquals(omCertId1, token1.decodeIdentifier().getOmCertSerialId()); + assertEquals(secretKey1.getId().toString(), token1.decodeIdentifier().getSecretKeyId()); // Renew delegation token long expiryTime = omClient.renewDelegationToken(token1); assertThat(expiryTime).isGreaterThan(0); - // Wait for OM certificate to renew - LambdaTestUtils.await(certLifetime, 100, () -> - !StringUtils.equals(token1.decodeIdentifier().getOmCertSerialId(), - omClient.getDelegationToken(new Text("om")) - .decodeIdentifier().getOmCertSerialId())); - String omCertId2 = - certClient.getCertificate().getSerialNumber().toString(); - assertNotEquals(omCertId1, omCertId2); + // Rotate secret key + secretKeyClient.rotate(); + ManagedSecretKey secretKey2 = secretKeyClient.getCurrentSecretKey(); + assertNotEquals(secretKey1.getId(), secretKey2.getId()); // Get a new delegation token Token token2 = omClient.getDelegationToken( new Text("om")); - assertEquals(omCertId2, token2.decodeIdentifier().getOmCertSerialId()); + assertEquals(secretKey2.getId().toString(), token2.decodeIdentifier().getSecretKeyId()); - // Because old certificate is still valid, so renew old token will succeed + // Because old secret key is still valid, so renew old token will succeed expiryTime = omClient.renewDelegationToken(token1); assertThat(expiryTime) .isGreaterThan(0) - .isLessThan(omCert.getNotAfter().getTime()); + .isLessThan(secretKey2.getExpiryTime().toEpochMilli()); } finally { if (scm != null) { scm.stop(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumesSecure.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumesSecure.java index 72f1c3374b2..6c7cd89109e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumesSecure.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerListVolumesSecure.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.client.SecretKeyTestClient; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.protocolPB.OmTransportFactory; @@ -201,6 +202,7 @@ private void setupEnvironment(boolean aclEnabled, om.setScmTopologyClient(new ScmTopologyClient( new ScmBlockLocationTestingClient(null, null, 0))); om.setCertClient(new CertificateClientTestImpl(conf)); + om.setSecretKeyClient(new SecretKeyTestClient()); om.start(); // Get OM client diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 9971506fa95..92c2b6b4cc5 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1469,7 +1469,8 @@ message OMTokenProto { optional string accessKeyId = 12; optional string signature = 13; optional string strToSign = 14; - optional string omServiceId = 15; + optional string omServiceId = 15 [deprecated = true]; + optional string secretKeyId = 16; } message SecretKeyProto { diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/TokenIdentifierCodec.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/TokenIdentifierCodec.java index edf65ae2247..84203b1f65a 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/TokenIdentifierCodec.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/TokenIdentifierCodec.java @@ -48,10 +48,10 @@ public Class getTypeClass() { } @Override - public byte[] toPersistedFormat(OzoneTokenIdentifier object) { + public byte[] toPersistedFormat(OzoneTokenIdentifier object) throws IOException { Preconditions .checkNotNull(object, "Null object can't be converted to byte array."); - return object.toUniqueSerializedKey(); + return object.toProtoBuf().toByteArray(); } @Override @@ -60,11 +60,11 @@ public OzoneTokenIdentifier fromPersistedFormat(byte[] rawData) Preconditions.checkNotNull(rawData, "Null byte array can't converted to real object."); try { - OzoneTokenIdentifier object = OzoneTokenIdentifier.newInstance(); - return object.fromUniqueSerializedKey(rawData); + return OzoneTokenIdentifier.readProtoBuf(rawData); } catch (IOException ex) { try { - return OzoneTokenIdentifier.readProtoBuf(rawData); + OzoneTokenIdentifier object = OzoneTokenIdentifier.newInstance(); + return object.fromUniqueSerializedKey(rawData); } catch (InvalidProtocolBufferException e) { throw new IllegalArgumentException( "Can't encode the the raw data from the byte array", e); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 705f7c9a01b..7ee32e840dd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -87,6 +87,8 @@ import org.apache.hadoop.hdds.scm.client.ScmTopologyClient; import org.apache.hadoop.hdds.scm.ha.SCMHAUtils; import org.apache.hadoop.hdds.scm.net.NetworkTopology; +import org.apache.hadoop.hdds.security.symmetric.DefaultSecretKeyClient; +import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient; import org.apache.hadoop.hdds.server.OzoneAdmins; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.Table.KeyValue; @@ -120,8 +122,6 @@ import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.hdds.security.SecurityConfig; -import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient; -import org.apache.hadoop.hdds.security.symmetric.DefaultSecretKeySignerClient; import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSecretManager; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl; @@ -371,7 +371,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl private OzoneDelegationTokenSecretManager delegationTokenMgr; private OzoneBlockTokenSecretManager blockTokenMgr; private CertificateClient certClient; - private SecretKeySignerClient secretKeyClient; + private SecretKeyClient secretKeyClient; private ScmTopologyClient scmTopologyClient; private final Text omRpcAddressTxt; private OzoneConfiguration configuration; @@ -671,8 +671,8 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) SecretKeyProtocol secretKeyProtocol = HddsServerUtil.getSecretKeyClientForOm(conf); - secretKeyClient = new DefaultSecretKeySignerClient(secretKeyProtocol, - omNodeDetails.threadNamePrefix()); + secretKeyClient = DefaultSecretKeyClient.create( + conf, secretKeyProtocol, omNodeDetails.threadNamePrefix()); } serviceInfo = new ServiceInfoProvider(secConfig, this, certClient, testSecureOmFlag); @@ -1085,6 +1085,7 @@ private OzoneDelegationTokenSecretManager createDelegationTokenSecretManager( .setOzoneManager(this) .setS3SecretManager(s3SecretManager) .setCertificateClient(certClient) + .setSecretKeyClient(secretKeyClient) .setOmServiceId(omNodeDetails.getServiceId()) .build(); } @@ -1127,7 +1128,7 @@ public void startSecretManager() { throw new UncheckedIOException(e); } - if (secConfig.isBlockTokenEnabled() && blockTokenMgr != null) { + if (secConfig.isSecurityEnabled()) { LOG.info("Starting secret key client."); try { secretKeyClient.start(configuration); @@ -1180,10 +1181,14 @@ public NetworkTopology getClusterMap() { * without fully setting up a working secure cluster. */ @VisibleForTesting - public void setSecretKeyClient( - SecretKeySignerClient secretKeyClient) { + public void setSecretKeyClient(SecretKeyClient secretKeyClient) { this.secretKeyClient = secretKeyClient; - blockTokenMgr.setSecretKeyClient(secretKeyClient); + if (blockTokenMgr != null) { + blockTokenMgr.setSecretKeyClient(secretKeyClient); + } + if (delegationTokenMgr != null) { + delegationTokenMgr.setSecretKeyClient(secretKeyClient); + } } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java index 5a62a7cfc62..e5d9901fda1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java @@ -46,7 +46,8 @@ public enum OMLayoutFeature implements LayoutFeature { FILESYSTEM_SNAPSHOT(5, "Ozone version supporting snapshot"), QUOTA(6, "Ozone quota re-calculate"), - HBASE_SUPPORT(7, "Full support of hsync, lease recovery and listOpenFiles APIs for HBase"); + HBASE_SUPPORT(7, "Full support of hsync, lease recovery and listOpenFiles APIs for HBase"), + DELEGATION_TOKEN_SYMMETRIC_SIGN(8, "Delegation token signed by symmetric key"); /////////////////////////////// ///////////////////////////// // Example OM Layout Feature with Actions diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java index a6fe61eb480..420cb6c6dcb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java @@ -25,13 +25,19 @@ import java.security.cert.X509Certificate; import java.util.Iterator; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.security.OzoneSecretManager; import org.apache.hadoop.hdds.security.SecurityConfig; +import org.apache.hadoop.hdds.security.exception.SCMSecurityException; +import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; +import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.security.x509.exception.CertificateException; import org.apache.hadoop.io.Text; @@ -41,6 +47,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMLeaderNotReadyException; import org.apache.hadoop.ozone.om.exceptions.OMNotLeaderException; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.security.OzoneSecretStore.OzoneManagerSecretState; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier.TokenInfo; import org.apache.hadoop.security.AccessControlException; @@ -64,7 +71,7 @@ public class OzoneDelegationTokenSecretManager extends OzoneSecretManager { - private static final Logger LOG = LoggerFactory + public static final Logger LOG = LoggerFactory .getLogger(OzoneDelegationTokenSecretManager.class); private final Map currentTokens; private final OzoneSecretStore store; @@ -73,6 +80,7 @@ public class OzoneDelegationTokenSecretManager private final long tokenRemoverScanInterval; private final String omServiceId; private final OzoneManager ozoneManager; + private SecretKeyClient secretKeyClient; /** * If the delegation token update thread holds this lock, it will not get @@ -100,8 +108,8 @@ public OzoneDelegationTokenSecretManager(Builder b) throws IOException { isRatisEnabled = b.ozoneConf.getBoolean( OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT); + this.secretKeyClient = b.secretKeyClient; loadTokenSecretState(store.loadState()); - } /** @@ -117,6 +125,7 @@ public static class Builder { private CertificateClient certClient; private String omServiceId; private OzoneManager ozoneManager; + private SecretKeyClient secretKeyClient; public OzoneDelegationTokenSecretManager build() throws IOException { return new OzoneDelegationTokenSecretManager(this); @@ -157,6 +166,11 @@ public Builder setCertificateClient(CertificateClient certificateClient) { return this; } + public Builder setSecretKeyClient(SecretKeyClient client) { + this.secretKeyClient = client; + return this; + } + public Builder setOmServiceId(String serviceId) { this.omServiceId = serviceId; return this; @@ -195,9 +209,15 @@ public Token createToken(Text owner, Text renewer, OzoneTokenIdentifier identifier = createIdentifier(owner, renewer, realUser); updateIdentifierDetails(identifier); - - byte[] password = createPassword(identifier.getBytes(), - getCurrentKey().getPrivateKey()); + byte[] password; + if (ozoneManager.getVersionManager().isAllowed(OMLayoutFeature.DELEGATION_TOKEN_SYMMETRIC_SIGN)) { + ManagedSecretKey currentSecretKey = secretKeyClient.getCurrentSecretKey(); + identifier.setSecretKeyId(currentSecretKey.getId().toString()); + password = currentSecretKey.sign(identifier.getBytes()); + } else { + identifier.setOmCertSerialId(getCertSerialId()); + password = createPassword(identifier.getBytes(), getCurrentKey().getPrivateKey()); + } long expiryTime = identifier.getIssueDate() + getTokenRenewInterval(); // For HA ratis will take care of updating. @@ -252,7 +272,6 @@ private void updateIdentifierDetails(OzoneTokenIdentifier identifier) { identifier.setMasterKeyId(getCurrentKey().getKeyId()); identifier.setSequenceNumber(sequenceNum); identifier.setMaxDate(now + getTokenMaxLifetime()); - identifier.setOmCertSerialId(getCertSerialId()); identifier.setOmServiceId(getOmServiceId()); } @@ -433,9 +452,29 @@ private TokenInfo validateToken(OzoneTokenIdentifier identifier) /** * Validates if given hash is valid. + * HDDS-8829 changes the delegation token from sign by OM's RSA private key to secret key supported by SCM. + * The default delegation token lifetime is 7 days. + * In the 7 days period after OM is upgraded from version without HDDS-8829 to version with HDDS-8829, tokens + * signed by RSA private key, and tokens signed by secret key will coexist. After 7 days, there will be only + * tokens signed by secrete key still valid. Following logic will handle both types of tokens. */ public boolean verifySignature(OzoneTokenIdentifier identifier, byte[] password) { + String secretKeyId = identifier.getSecretKeyId(); + if (StringUtils.isNotEmpty(secretKeyId)) { + try { + ManagedSecretKey verifyKey = secretKeyClient.getSecretKey(UUID.fromString(secretKeyId)); + return verifyKey.isValidSignature(identifier.getBytes(), password); + } catch (SCMSecurityException e) { + LOG.error("verifySignature for identifier {} failed", identifier, e); + return false; + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Verify an asymmetric key signed Token {}", identifier); + } + X509Certificate signerCert; try { signerCert = getCertClient().getCertificate( @@ -511,6 +550,14 @@ private byte[] validateS3AuthInfo(OzoneTokenIdentifier identifier) } + /** + * Load delegation tokens from DB into memory. + * HDDS-8829 changes the delegation token from sign by OM's RSA private key to secret key supported by SCM. + * The default delegation token lifetime is 7 days. After OM is upgraded from version without HDDS-8829 to + * version with HDDS-8829 and restarts, tokens signed by RSA private key will be loaded from DB into memory. + * Next OM restarts, if after 7 days, there will be only tokens signed by secret key loaded into memory. + * Both types of token loading should be supported. + */ private void loadTokenSecretState( OzoneManagerSecretState state) throws IOException { LOG.info("Loading token state into token manager."); @@ -528,8 +575,17 @@ private void addPersistedDelegationToken(OzoneTokenIdentifier identifier, "Can't add persisted delegation token to a running SecretManager."); } - byte[] password = createPassword(identifier.getBytes(), - getCertClient().getPrivateKey()); + byte[] password; + if (StringUtils.isNotEmpty(identifier.getSecretKeyId())) { + ManagedSecretKey signKey = secretKeyClient.getSecretKey(UUID.fromString(identifier.getSecretKeyId())); + password = signKey.sign(identifier.getBytes()); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Load an asymmetric key signed Token {}", identifier); + } + password = createPassword(identifier.getBytes(), getCertClient().getPrivateKey()); + } + if (identifier.getSequenceNumber() > getDelegationTokenSeqNum()) { setDelegationTokenSeqNum(identifier.getSequenceNumber()); } @@ -588,6 +644,11 @@ public void stop() throws IOException { } } + @VisibleForTesting + public void setSecretKeyClient(SecretKeyClient client) { + this.secretKeyClient = client; + } + /** * Remove expired delegation tokens from cache and persisted store. */ diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/SecretKeyTestClient.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/SecretKeyTestClient.java new file mode 100644 index 00000000000..32ef5988e10 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/SecretKeyTestClient.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.security; + +import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; +import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +/** + * Test implementation of {@link SecretKeyClient}. + */ +public class SecretKeyTestClient implements SecretKeyClient { + private final Map keysMap = new HashMap<>(); + private ManagedSecretKey current; + + public SecretKeyTestClient() { + rotate(); + } + + public void rotate() { + this.current = generateKey(); + keysMap.put(current.getId(), current); + } + + @Override + public ManagedSecretKey getCurrentSecretKey() { + return current; + } + + @Override + public ManagedSecretKey getSecretKey(UUID id) { + return keysMap.get(id); + } + + private ManagedSecretKey generateKey() { + KeyGenerator keyGen = null; + try { + keyGen = KeyGenerator.getInstance("HmacSHA256"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("Should never happen", e); + } + SecretKey secretKey = keyGen.generateKey(); + return new ManagedSecretKey( + UUID.randomUUID(), + Instant.now(), + Instant.now().plus(Duration.ofHours(1)), + secretKey + ); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java index d94f59b8fb8..c0fdb7a8c21 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java @@ -24,7 +24,6 @@ import java.security.KeyPair; import java.security.PrivateKey; import java.security.PublicKey; -import java.security.Signature; import java.security.cert.CertPath; import java.security.cert.CertificateFactory; import java.security.cert.X509Certificate; @@ -35,6 +34,8 @@ import com.google.common.collect.ImmutableList; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.security.SecurityConfig; +import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; +import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.io.Text; @@ -50,6 +51,8 @@ import org.apache.hadoop.ozone.om.exceptions.OMLeaderNotReadyException; import org.apache.hadoop.ozone.om.exceptions.OMNotLeaderException; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; +import org.apache.hadoop.ozone.upgrade.LayoutFeature; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.security.token.SecretManager; @@ -64,11 +67,14 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import org.slf4j.event.Level; +import org.apache.ozone.test.GenericTestUtils; import org.apache.ratis.protocol.RaftPeerId; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -87,6 +93,7 @@ public class TestOzoneDelegationTokenSecretManager { private OzoneDelegationTokenSecretManager secretManager; private SecurityConfig securityConfig; private OMCertificateClient certificateClient; + private SecretKeyClient secretKeyClient; private long expiryTime; private Text serviceRpcAdd; private OzoneConfiguration conf; @@ -102,6 +109,7 @@ public void setUp() throws Exception { securityConfig = new SecurityConfig(conf); certificateClient = setupCertificateClient(); certificateClient.init(); + secretKeyClient = new SecretKeyTestClient(); expiryTime = Time.monotonicNow() + 60 * 60 * 24; serviceRpcAdd = new Text("localhost"); final Map s3Secrets = new HashMap<>(); @@ -112,6 +120,9 @@ public void setUp() throws Exception { om = mock(OzoneManager.class); OMMetadataManager metadataManager = new OmMetadataManagerImpl(conf, om); when(om.getMetadataManager()).thenReturn(metadataManager); + OMLayoutVersionManager versionManager = mock(OMLayoutVersionManager.class); + when(versionManager.isAllowed(any(LayoutFeature.class))).thenReturn(true); + when(om.getVersionManager()).thenReturn(versionManager); s3SecretManager = new S3SecretLockedManager( new S3SecretManagerImpl(new S3SecretStoreMap(s3Secrets), mock(S3SecretCache.class)), @@ -368,12 +379,28 @@ public void testVerifySignatureSuccess() throws Exception { expiryTime, TOKEN_REMOVER_SCAN_INTERVAL); secretManager.start(certificateClient); OzoneTokenIdentifier id = new OzoneTokenIdentifier(); + id.setMaxDate(Time.now() + 60 * 60 * 24); + id.setOwner(new Text("test")); + id.setSecretKeyId(secretKeyClient.getCurrentSecretKey().getId().toString()); + assertTrue(secretManager.verifySignature(id, secretKeyClient.getCurrentSecretKey().sign(id.getBytes()))); + } + + @Test + public void testVerifyAsymmetricSignatureSuccess() throws Exception { + GenericTestUtils.setLogLevel(OzoneDelegationTokenSecretManager.LOG, Level.DEBUG); + GenericTestUtils.LogCapturer logCapturer = + GenericTestUtils.LogCapturer.captureLogs(OzoneDelegationTokenSecretManager.LOG); + secretManager = createSecretManager(conf, TOKEN_MAX_LIFETIME, + expiryTime, TOKEN_REMOVER_SCAN_INTERVAL); + secretManager.start(certificateClient); + OzoneTokenIdentifier id = new OzoneTokenIdentifier(); id.setOmCertSerialId(certificateClient.getCertificate() .getSerialNumber().toString()); id.setMaxDate(Time.now() + 60 * 60 * 24); id.setOwner(new Text("test")); - assertTrue(secretManager.verifySignature(id, - certificateClient.signData(id.getBytes()))); + assertTrue(secretManager.verifySignature(id, certificateClient.signData(id.getBytes()))); + assertTrue(logCapturer.getOutput().contains("Verify an asymmetric key signed Token")); + logCapturer.stopCapturing(); } @Test @@ -461,12 +488,9 @@ public void testValidateS3AUTHINFOFailure() throws Exception { * Validate hash using public key of KeyPair. */ private void validateHash(byte[] hash, byte[] identifier) throws Exception { - Signature rsaSignature = - Signature.getInstance(securityConfig.getSignatureAlgo(), - securityConfig.getProvider()); - rsaSignature.initVerify(certificateClient.getPublicKey()); - rsaSignature.update(identifier); - assertTrue(rsaSignature.verify(hash)); + OzoneTokenIdentifier ozoneTokenIdentifier = OzoneTokenIdentifier.readProtoBuf(identifier); + ManagedSecretKey verifyKey = secretKeyClient.getSecretKey(UUID.fromString(ozoneTokenIdentifier.getSecretKeyId())); + verifyKey.isValidSignature(identifier, hash); } /** @@ -485,6 +509,7 @@ private void validateHash(byte[] hash, byte[] identifier) throws Exception { .setS3SecretManager(s3SecretManager) .setCertificateClient(certificateClient) .setOmServiceId(OzoneConsts.OM_SERVICE_ID_DEFAULT) + .setSecretKeyClient(secretKeyClient) .build(); } }