Skip to content

Commit

Permalink
Revert "automatic recovery in cert client code draft"
Browse files Browse the repository at this point in the history 
This reverts commit 166883a.
  • Loading branch information
Sadanand Shenoy committed Aug 29, 2024
1 parent 166883a commit 40ffb3a
Show file tree
Hide file tree
Showing 4 changed files with 110 additions and 103 deletions.
5 changes: 0 additions & 5 deletions hadoop-hdds/framework/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>ozone-tools</artifactId>
<version>1.5.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-managed-rocksdb</artifactId>
Expand Down
38 changes: 8 additions & 30 deletions ...in/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,6 @@
import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateSignRequest;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.OzoneSecurityUtil;
import org.apache.hadoop.ozone.repair.RecoverSCMCertificate;
import org.bouncycastle.cert.X509CertificateHolder;
import org.bouncycastle.pkcs.PKCS10CertificationRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -81,21 +78,19 @@ public class SCMCertificateClient extends DefaultCertificateClient {
private ExecutorService executorService;
private boolean isPrimarySCM = false;
private Consumer<String> saveCertIdCallback;
private String scmDbPath;

@SuppressWarnings("parameternumber")
public SCMCertificateClient(SecurityConfig securityConfig,
SCMSecurityProtocolClientSideTranslatorPB scmClient,
String scmId, String clusterId, String scmCertId, String hostname,
boolean isPrimarySCM, Consumer<String> saveCertId,String scmDbPath) {
boolean isPrimarySCM, Consumer<String> saveCertId) {
super(securityConfig, scmClient, LOG, scmCertId, COMPONENT_NAME,
HddsUtils.threadNamePrefix(scmId), saveCertId, null);
this.scmId = scmId;
this.cId = clusterId;
this.scmHostname = hostname;
this.isPrimarySCM = isPrimarySCM;
this.saveCertIdCallback = saveCertId;
this.scmDbPath = scmDbPath;
}

public SCMCertificateClient(SecurityConfig securityConfig,
Expand Down Expand Up @@ -134,24 +129,20 @@ public SCMCertificateClient(
component);
}

public String getScmDbPath() {
return scmDbPath;
}

/**
* Returns a CSR builder that can be used to creates a Certificate signing
* request.
*
* @return CertificateSignRequest.Builder
*/
public CertificateSignRequest.Builder getCSRBuilder()
throws CertificateException {
public CertificateSignRequest.Builder configureCSRBuilder()
throws SCMSecurityException {
String subject = SCM_SUB_CA_PREFIX + scmHostname;

LOG.info("Creating csr for SCM->hostName:{},scmId:{},clusterId:{}," +
"subject:{}", scmHostname, scmId, cId, subject);

return super.getCSRBuilder()
return super.configureCSRBuilder()
.setSubject(subject)
.setScmID(scmId)
.setClusterID(cId)
Expand Down Expand Up @@ -286,28 +277,15 @@ public synchronized void close() throws IOException {
@Override
protected void recoverStateIfNeeded(InitResponse state) throws IOException {
LOG.info("Init response: {}", state);
// make this check based on a config?
boolean checkDB = false;
switch (state) {
case SUCCESS:
LOG.info("Initialization successful.");
break;
case GETCERT:
boolean checkDbSuccess = false;
if (checkDB) {
try {
checkDbSuccess = RecoverSCMCertificate.getAndStoreCerts(getSecurityConfig(), getScmDbPath());
} catch (Exception e) {
LOG.error("Couldn't retrieve certs from DB");
throw new IOException(e.getMessage());
}
}
if (!checkDbSuccess) {
if (!isPrimarySCM) {
getRootCASignedSCMCert();
} else {
getPrimarySCMSelfSignedCert();
}
if (!isPrimarySCM) {
getRootCASignedSCMCert();
} else {
getPrimarySCMSelfSignedCert();
}
LOG.info("Successfully stored SCM signed certificate.");
break;
Expand Down
11 changes: 2 additions & 9 deletions hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@
import org.apache.hadoop.hdds.security.x509.certificate.authority.profile.PKIProfile;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient;
import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.conf.RaftProperties;
Expand All @@ -45,11 +43,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.math.BigInteger;
import java.net.InetAddress;
import java.nio.file.Paths;
import java.security.cert.X509Certificate;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
Expand Down Expand Up @@ -82,10 +78,7 @@ public static void initializeSecurity(SCMStorageConfig scmStorageConfig,
OzoneConfiguration conf, String scmHostname, boolean primaryscm)
throws IOException {
LOG.info("Initializing secure StorageContainerManager.");
File scmDbDir = ServerUtils.getScmDbDir(conf);
String dbPath = Paths.get(scmDbDir.getAbsolutePath(), OzoneConsts.SCM_DB_NAME)
.toFile().getAbsolutePath();
LOG.info("SCM DB Path is : {}");

SecurityConfig securityConfig = new SecurityConfig(conf);
SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient =
getScmSecurityClientWithFixedDuration(conf);
Expand All @@ -100,7 +93,7 @@ public static void initializeSecurity(SCMStorageConfig scmStorageConfig,
LOG.error("Failed to set new certificate ID", e);
throw new RuntimeException("Failed to set new certificate ID");
}
}, dbPath)) {
})) {
certClient.initWithRecovery();
}
}
Expand Down
159 changes: 100 additions & 59 deletions hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.repair;
package org.apache.hadoop.ozone.debug;

import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
import org.apache.hadoop.hdds.cli.SubcommandWithParent;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.security.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.authority.CAType;
import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient;
Expand All @@ -27,16 +28,19 @@
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.debug.RocksDBUtils;
import org.bouncycastle.jcajce.provider.asymmetric.x509.CertificateFactory;
import org.kohsuke.MetaInfServices;
import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.RocksDBException;
import picocli.CommandLine;

import java.io.IOException;
import java.io.PrintWriter;
import java.math.BigInteger;
import java.net.InetAddress;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.security.cert.CertPath;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
Expand All @@ -46,19 +50,92 @@
import java.util.ArrayList;
import java.util.Optional;
import java.util.Arrays;
import java.util.concurrent.Callable;

import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.VALID_SCM_CERTS;
import static org.apache.hadoop.hdds.security.x509.certificate.client.DefaultCertificateClient.CERT_FILE_NAME_FORMAT;
import static org.apache.hadoop.hdds.utils.db.DBDefinition.LOG;
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.removeTrailingSlashIfNeeded;

/**
* In case of accidental deletion of SCM certificates from local storage,
* this tool restores the certs that are persisted into the SCM DB.
* Note that this will only work if the SCM has persisted certs in its RocksDB
* and the private keys are intact and not lost. If private keys of the SCM are
* lost, this tool is not of much use.
* Note that this will only work if the SCM has persisted certs in its RocksDB.
*/
public class RecoverSCMCertificate {
@CommandLine.Command(
name = "cert-recover",
description = "Recover Deleted SCM Certificate from RocksDB")
@MetaInfServices(SubcommandWithParent.class)
public class RecoverSCMCertificate implements Callable<Void>, SubcommandWithParent {

@CommandLine.Option(names = {"--db"},
required = true,
description = "SCM DB Path")
private String dbPath;

@CommandLine.ParentCommand
private OzoneDebug parent;

@CommandLine.Spec
private CommandLine.Model.CommandSpec spec;

@Override
public Class<?> getParentType() {
return OzoneDebug.class;
}

private PrintWriter err() {
return spec.commandLine().getErr();
}

private PrintWriter out() {
return spec.commandLine().getOut();
}

@Override
public Void call() throws Exception {
dbPath = removeTrailingSlashIfNeeded(dbPath);
String tableName = VALID_SCM_CERTS.getName();
DBDefinition dbDefinition =
DBDefinitionFactory.getDefinition(Paths.get(dbPath), new OzoneConfiguration());
if (dbDefinition == null) {
throw new Exception("Error: Incorrect DB Path");
}
DBColumnFamilyDefinition columnFamilyDefinition =
getDbColumnFamilyDefinition(tableName, dbDefinition);

try {
List<ColumnFamilyDescriptor> cfDescList = RocksDBUtils.getColumnFamilyDescriptors(dbPath);
final List<ColumnFamilyHandle> cfHandleList = new ArrayList<>();
byte[] tableNameBytes = tableName.getBytes(StandardCharsets.UTF_8);
ColumnFamilyHandle cfHandle = null;
try (ManagedRocksDB db = ManagedRocksDB.openReadOnly(dbPath, cfDescList,
cfHandleList)) {
cfHandle = getColumnFamilyHandle(cfHandleList, tableNameBytes);

Map<BigInteger, X509Certificate> allCerts = getAllCerts(columnFamilyDefinition, cfHandle, db);
out().println("All Certs in DB : " + allCerts.keySet());
String hostName = InetAddress.getLocalHost().getCanonicalHostName();
out().println("Host: " + hostName);

X509Certificate subCertificate = getSubCertificate(allCerts, hostName);
X509Certificate rootCertificate = getRootCertificate(allCerts);

out().println("Sub cert serialID for this host: " + subCertificate.getSerialNumber().toString());
out().println("Root cert serialID: " + rootCertificate.getSerialNumber().toString());

boolean isRootCA = false;

String caPrincipal = rootCertificate.getSubjectDN().getName();
if (caPrincipal.contains(hostName)) {
isRootCA = true;
}
storeCerts(subCertificate, rootCertificate, isRootCA);
}
} catch (RocksDBException | CertificateException exception) {
err().print("Failed to recover scm cert");
}
return null;
}

private static ColumnFamilyHandle getColumnFamilyHandle(
List<ColumnFamilyHandle> cfHandleList, byte[] tableNameBytes) throws Exception {
Expand All @@ -80,7 +157,10 @@ private static X509Certificate getRootCertificate(
Optional<X509Certificate> cert = allCerts.values().stream().filter(
c -> c.getSubjectDN().getName()
.contains(OzoneConsts.SCM_ROOT_CA_PREFIX)).findFirst();
return cert.orElse(null);
if (!cert.isPresent()) {
throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet());
}
return cert.get();
}


Expand All @@ -90,53 +170,13 @@ private static X509Certificate getSubCertificate(
c -> c.getSubjectDN().getName()
.contains(OzoneConsts.SCM_SUB_CA_PREFIX) && c.getSubjectDN()
.getName().contains(hostName)).findFirst();
return cert.orElse(null);
}


public static boolean getAndStoreCerts(SecurityConfig conf,String dbPath)
throws Exception {
DBDefinition dbDefinition = new SCMDBDefinition();
String tableName = VALID_SCM_CERTS.getName();
DBColumnFamilyDefinition columnFamilyDefinition =
getDbColumnFamilyDefinition(tableName, dbDefinition);
List<ColumnFamilyDescriptor> cfDescList = RocksDBUtils.getColumnFamilyDescriptors(dbPath);
final List<ColumnFamilyHandle> cfHandleList = new ArrayList<>();
byte[] tableNameBytes = tableName.getBytes(StandardCharsets.UTF_8);
ColumnFamilyHandle cfHandle = null;
try (ManagedRocksDB db = ManagedRocksDB.openReadOnly(dbPath, cfDescList,
cfHandleList)) {
cfHandle = getColumnFamilyHandle(cfHandleList, tableNameBytes);

Map<BigInteger, X509Certificate> allCerts = getAllCerts(columnFamilyDefinition, cfHandle, db);
LOG.info("All Certs in DB : " + allCerts.keySet());
String hostName = InetAddress.getLocalHost().getCanonicalHostName();
LOG.info("Host: " + hostName);

X509Certificate subCertificate = getSubCertificate(allCerts, hostName);
X509Certificate rootCertificate = getRootCertificate(allCerts);

boolean containsCerts = subCertificate!=null && rootCertificate!=null;

if (containsCerts) {
LOG.info("Sub cert serialID for this host: " + subCertificate.getSerialNumber()
.toString());
LOG.info("Root cert serialID: " + rootCertificate.getSerialNumber().toString());

boolean isRootCA = false;

String caPrincipal = rootCertificate.getSubjectDN().getName();
if (caPrincipal.contains(hostName)) {
isRootCA = true;
}
storeCerts(subCertificate, rootCertificate, isRootCA, conf);
}

return containsCerts;
if (!cert.isPresent()) {
throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet());
}
return cert.get();
}

public static Map<BigInteger, X509Certificate> getAllCerts(
private static Map<BigInteger, X509Certificate> getAllCerts(
DBColumnFamilyDefinition columnFamilyDefinition,
ColumnFamilyHandle cfHandle, ManagedRocksDB db) throws IOException, RocksDBException {
Map<BigInteger, X509Certificate> allCerts = new HashMap<>();
Expand Down Expand Up @@ -165,13 +205,14 @@ private static DBColumnFamilyDefinition getDbColumnFamilyDefinition(
return columnFamilyDefinition;
}

private static void storeCerts(X509Certificate scmCertificate,
X509Certificate rootCertificate, boolean isRootCA, SecurityConfig securityConfig)
private void storeCerts(X509Certificate scmCertificate,
X509Certificate rootCertificate, boolean isRootCA)
throws CertificateException, IOException {
SecurityConfig securityConfig = new SecurityConfig(parent.getOzoneConf());
CertificateCodec certCodec =
new CertificateCodec(securityConfig, SCMCertificateClient.COMPONENT_NAME);

LOG.info("Writing certs to path : " + certCodec.getLocation().toString());
out().println("Writing certs to path : " + certCodec.getLocation().toString());

CertPath certPath = addRootCertInPath(scmCertificate, rootCertificate);
CertPath rootCertPath = getRootCertPath(rootCertificate);
Expand All @@ -191,13 +232,13 @@ private static void storeCerts(X509Certificate scmCertificate,
if (isRootCA) {
CertificateCodec rootCertCodec =
new CertificateCodec(securityConfig, OzoneConsts.SCM_ROOT_CA_COMPONENT_NAME);
LOG.info("Writing root certs to path : " + rootCertCodec.getLocation().toString());
out().println("Writing root certs to path : " + rootCertCodec.getLocation().toString());
rootCertCodec.writeCertificate(rootCertCodec.getLocation().toAbsolutePath(),
securityConfig.getCertificateFileName(), encodedRootCert);
}
}

public static CertPath addRootCertInPath(X509Certificate scmCert,
public CertPath addRootCertInPath(X509Certificate scmCert,
X509Certificate rootCert) throws CertificateException {
ArrayList<X509Certificate> updatedList = new ArrayList<>();
updatedList.add(scmCert);
Expand All @@ -206,7 +247,7 @@ public static CertPath addRootCertInPath(X509Certificate scmCert,
return factory.engineGenerateCertPath(updatedList);
}

public static CertPath getRootCertPath(X509Certificate rootCert)
public CertPath getRootCertPath(X509Certificate rootCert)
throws CertificateException {
ArrayList<X509Certificate> updatedList = new ArrayList<>();
updatedList.add(rootCert);
Expand Down

0 comments on commit 40ffb3a

Please sign in to comment.