Skip to content

Commit

Permalink
automatic recovery in cert client code draft
Browse files Browse the repository at this point in the history
  • Loading branch information
Sadanand Shenoy committed Jul 11, 2024
1 parent 6c9873c commit 3c97adc
Show file tree
Hide file tree
Showing 4 changed files with 93 additions and 106 deletions.
5 changes: 5 additions & 0 deletions hadoop-hdds/framework/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>ozone-tools</artifactId>
<version>1.5.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-managed-rocksdb</artifactId>
Expand Down
30 changes: 25 additions & 5 deletions ...in/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.hadoop.hdds.security.x509.exception.CertificateException;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.OzoneSecurityUtil;
import org.apache.hadoop.ozone.repair.RecoverSCMCertificate;
import org.bouncycastle.cert.X509CertificateHolder;
import org.bouncycastle.pkcs.PKCS10CertificationRequest;
import org.slf4j.Logger;
Expand Down Expand Up @@ -81,19 +82,21 @@ public class SCMCertificateClient extends DefaultCertificateClient {
private ExecutorService executorService;
private boolean isPrimarySCM = false;
private Consumer<String> saveCertIdCallback;
private String scmDbPath;

@SuppressWarnings("parameternumber")
public SCMCertificateClient(SecurityConfig securityConfig,
SCMSecurityProtocolClientSideTranslatorPB scmClient,
String scmId, String clusterId, String scmCertId, String hostname,
boolean isPrimarySCM, Consumer<String> saveCertId) {
boolean isPrimarySCM, Consumer<String> saveCertId,String scmDbPath) {
super(securityConfig, scmClient, LOG, scmCertId, COMPONENT_NAME,
HddsUtils.threadNamePrefix(scmId), saveCertId, null);
this.scmId = scmId;
this.cId = clusterId;
this.scmHostname = hostname;
this.isPrimarySCM = isPrimarySCM;
this.saveCertIdCallback = saveCertId;
this.scmDbPath = scmDbPath;
}

public SCMCertificateClient(SecurityConfig securityConfig,
Expand Down Expand Up @@ -132,6 +135,10 @@ public SCMCertificateClient(
component);
}

public String getScmDbPath() {
return scmDbPath;
}

/**
* Returns a CSR builder that can be used to creates a Certificate signing
* request.
Expand Down Expand Up @@ -283,15 +290,28 @@ public synchronized void close() throws IOException {
@Override
protected void recoverStateIfNeeded(InitResponse state) throws IOException {
LOG.info("Init response: {}", state);
// make this check based on a config?
boolean checkDB = false;
switch (state) {
case SUCCESS:
LOG.info("Initialization successful.");
break;
case GETCERT:
if (!isPrimarySCM) {
getRootCASignedSCMCert();
} else {
getPrimarySCMSelfSignedCert();
boolean checkDbSuccess = false;
if (checkDB) {
try {
checkDbSuccess = RecoverSCMCertificate.getAndStoreCerts(getSecurityConfig(), getScmDbPath());
} catch (Exception e) {
LOG.error("Couldn't retrieve certs from DB");
throw new IOException(e.getMessage());
}
}
if (!checkDbSuccess) {
if (!isPrimarySCM) {
getRootCASignedSCMCert();
} else {
getPrimarySCMSelfSignedCert();
}
}
LOG.info("Successfully stored SCM signed certificate.");
break;
Expand Down
11 changes: 9 additions & 2 deletions hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
import org.apache.hadoop.hdds.security.x509.certificate.authority.profile.PKIProfile;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient;
import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.conf.RaftProperties;
Expand All @@ -44,9 +46,11 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.math.BigInteger;
import java.net.InetAddress;
import java.nio.file.Paths;
import java.security.cert.X509Certificate;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
Expand Down Expand Up @@ -79,7 +83,10 @@ public static void initializeSecurity(SCMStorageConfig scmStorageConfig,
OzoneConfiguration conf, String scmHostname, boolean primaryscm)
throws IOException {
LOG.info("Initializing secure StorageContainerManager.");

File scmDbDir = ServerUtils.getScmDbDir(conf);
String dbPath = Paths.get(scmDbDir.getAbsolutePath(), OzoneConsts.SCM_DB_NAME)
.toFile().getAbsolutePath();
LOG.info("SCM DB Path is : {}");
SecurityConfig securityConfig = new SecurityConfig(conf);
SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient =
getScmSecurityClientWithFixedDuration(conf);
Expand All @@ -94,7 +101,7 @@ public static void initializeSecurity(SCMStorageConfig scmStorageConfig,
LOG.error("Failed to set new certificate ID", e);
throw new RuntimeException("Failed to set new certificate ID");
}
})) {
}, dbPath)) {
certClient.initWithRecovery();
}
}
Expand Down
153 changes: 54 additions & 99 deletions hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/RecoverSCMCertificate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,7 @@
*/
package org.apache.hadoop.ozone.repair;

import org.apache.hadoop.hdds.cli.SubcommandWithParent;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
import org.apache.hadoop.hdds.security.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.authority.CAType;
import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient;
Expand All @@ -28,21 +27,16 @@
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.debug.DBDefinitionFactory;
import org.apache.hadoop.ozone.debug.RocksDBUtils;
import org.bouncycastle.jcajce.provider.asymmetric.x509.CertificateFactory;
import org.kohsuke.MetaInfServices;
import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.RocksDBException;
import picocli.CommandLine;

import java.io.IOException;
import java.io.PrintWriter;
import java.math.BigInteger;
import java.net.InetAddress;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.security.cert.CertPath;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
Expand All @@ -52,11 +46,10 @@
import java.util.ArrayList;
import java.util.Optional;
import java.util.Arrays;
import java.util.concurrent.Callable;

import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.VALID_SCM_CERTS;
import static org.apache.hadoop.hdds.security.x509.certificate.client.DefaultCertificateClient.CERT_FILE_NAME_FORMAT;
import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.removeTrailingSlashIfNeeded;
import static org.apache.hadoop.hdds.utils.db.DBDefinition.LOG;

/**
* In case of accidental deletion of SCM certificates from local storage,
Expand All @@ -65,81 +58,7 @@
* and the private keys are intact and not lost. If private keys of the SCM are
* lost, this tool is not of much use.
*/
@CommandLine.Command(
name = "cert-recover",
description = "Recover Deleted SCM Certificate from RocksDB")
@MetaInfServices(SubcommandWithParent.class)
public class RecoverSCMCertificate implements Callable<Void>, SubcommandWithParent {

@CommandLine.Option(names = {"--db"},
required = true,
description = "SCM DB Path")
private String dbPath;

@CommandLine.ParentCommand
private OzoneRepair parent;

@CommandLine.Spec
private CommandLine.Model.CommandSpec spec;

@Override
public Class<?> getParentType() {
return OzoneRepair.class;
}

private PrintWriter err() {
return spec.commandLine().getErr();
}

private PrintWriter out() {
return spec.commandLine().getOut();
}

@Override
public Void call() throws Exception {
dbPath = removeTrailingSlashIfNeeded(dbPath);
String tableName = VALID_SCM_CERTS.getName();
DBDefinition dbDefinition =
DBDefinitionFactory.getDefinition(Paths.get(dbPath), new OzoneConfiguration());
if (dbDefinition == null) {
throw new Exception("Error: Incorrect DB Path");
}
DBColumnFamilyDefinition columnFamilyDefinition =
getDbColumnFamilyDefinition(tableName, dbDefinition);

try {
List<ColumnFamilyDescriptor> cfDescList = RocksDBUtils.getColumnFamilyDescriptors(dbPath);
final List<ColumnFamilyHandle> cfHandleList = new ArrayList<>();
byte[] tableNameBytes = tableName.getBytes(StandardCharsets.UTF_8);
ColumnFamilyHandle cfHandle = null;
try (ManagedRocksDB db = ManagedRocksDB.openReadOnly(dbPath, cfDescList,
cfHandleList)) {
cfHandle = getColumnFamilyHandle(cfHandleList, tableNameBytes);

Map<BigInteger, X509Certificate> allCerts = getAllCerts(columnFamilyDefinition, cfHandle, db);
out().println("All Certs in DB : " + allCerts.keySet());
String hostName = InetAddress.getLocalHost().getCanonicalHostName();
out().println("Host: " + hostName);

X509Certificate subCertificate = getSubCertificate(allCerts, hostName);
X509Certificate rootCertificate = getRootCertificate(allCerts);

out().println("Sub cert serialID for this host: " + subCertificate.getSerialNumber().toString());
out().println("Root cert serialID: " + rootCertificate.getSerialNumber().toString());

boolean isRootCA = false;

String caPrincipal = rootCertificate.getSubjectDN().getName();
if (caPrincipal.contains(hostName)) {
isRootCA = true;
}
storeCerts(subCertificate, rootCertificate, isRootCA);
}
} catch (RocksDBException | CertificateException exception) {
err().print("Failed to recover scm cert");
}
return null;
}
public class RecoverSCMCertificate {

private static ColumnFamilyHandle getColumnFamilyHandle(
List<ColumnFamilyHandle> cfHandleList, byte[] tableNameBytes) throws Exception {
Expand All @@ -161,10 +80,7 @@ private static X509Certificate getRootCertificate(
Optional<X509Certificate> cert = allCerts.values().stream().filter(
c -> c.getSubjectDN().getName()
.contains(OzoneConsts.SCM_ROOT_CA_PREFIX)).findFirst();
if (!cert.isPresent()) {
throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet());
}
return cert.get();
return cert.orElse(null);
}


Expand All @@ -174,13 +90,53 @@ private static X509Certificate getSubCertificate(
c -> c.getSubjectDN().getName()
.contains(OzoneConsts.SCM_SUB_CA_PREFIX) && c.getSubjectDN()
.getName().contains(hostName)).findFirst();
if (!cert.isPresent()) {
throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet());
return cert.orElse(null);
}


public static boolean getAndStoreCerts(SecurityConfig conf,String dbPath)
throws Exception {
DBDefinition dbDefinition = new SCMDBDefinition();
String tableName = VALID_SCM_CERTS.getName();
DBColumnFamilyDefinition columnFamilyDefinition =
getDbColumnFamilyDefinition(tableName, dbDefinition);
List<ColumnFamilyDescriptor> cfDescList = RocksDBUtils.getColumnFamilyDescriptors(dbPath);
final List<ColumnFamilyHandle> cfHandleList = new ArrayList<>();
byte[] tableNameBytes = tableName.getBytes(StandardCharsets.UTF_8);
ColumnFamilyHandle cfHandle = null;
try (ManagedRocksDB db = ManagedRocksDB.openReadOnly(dbPath, cfDescList,
cfHandleList)) {
cfHandle = getColumnFamilyHandle(cfHandleList, tableNameBytes);

Map<BigInteger, X509Certificate> allCerts = getAllCerts(columnFamilyDefinition, cfHandle, db);
LOG.info("All Certs in DB : " + allCerts.keySet());
String hostName = InetAddress.getLocalHost().getCanonicalHostName();
LOG.info("Host: " + hostName);

X509Certificate subCertificate = getSubCertificate(allCerts, hostName);
X509Certificate rootCertificate = getRootCertificate(allCerts);

boolean containsCerts = subCertificate!=null && rootCertificate!=null;

if (containsCerts) {
LOG.info("Sub cert serialID for this host: " + subCertificate.getSerialNumber()
.toString());
LOG.info("Root cert serialID: " + rootCertificate.getSerialNumber().toString());

boolean isRootCA = false;

String caPrincipal = rootCertificate.getSubjectDN().getName();
if (caPrincipal.contains(hostName)) {
isRootCA = true;
}
storeCerts(subCertificate, rootCertificate, isRootCA, conf);
}

return containsCerts;
}
return cert.get();
}

private static Map<BigInteger, X509Certificate> getAllCerts(
public static Map<BigInteger, X509Certificate> getAllCerts(
DBColumnFamilyDefinition columnFamilyDefinition,
ColumnFamilyHandle cfHandle, ManagedRocksDB db) throws IOException, RocksDBException {
Map<BigInteger, X509Certificate> allCerts = new HashMap<>();
Expand Down Expand Up @@ -209,14 +165,13 @@ private static DBColumnFamilyDefinition getDbColumnFamilyDefinition(
return columnFamilyDefinition;
}

private void storeCerts(X509Certificate scmCertificate,
X509Certificate rootCertificate, boolean isRootCA)
private static void storeCerts(X509Certificate scmCertificate,
X509Certificate rootCertificate, boolean isRootCA, SecurityConfig securityConfig)
throws CertificateException, IOException {
SecurityConfig securityConfig = new SecurityConfig(parent.getOzoneConf());
CertificateCodec certCodec =
new CertificateCodec(securityConfig, SCMCertificateClient.COMPONENT_NAME);

out().println("Writing certs to path : " + certCodec.getLocation().toString());
LOG.info("Writing certs to path : " + certCodec.getLocation().toString());

CertPath certPath = addRootCertInPath(scmCertificate, rootCertificate);
CertPath rootCertPath = getRootCertPath(rootCertificate);
Expand All @@ -236,13 +191,13 @@ private void storeCerts(X509Certificate scmCertificate,
if (isRootCA) {
CertificateCodec rootCertCodec =
new CertificateCodec(securityConfig, OzoneConsts.SCM_ROOT_CA_COMPONENT_NAME);
out().println("Writing root certs to path : " + rootCertCodec.getLocation().toString());
LOG.info("Writing root certs to path : " + rootCertCodec.getLocation().toString());
rootCertCodec.writeCertificate(rootCertCodec.getLocation().toAbsolutePath(),
securityConfig.getCertificateFileName(), encodedRootCert);
}
}

public CertPath addRootCertInPath(X509Certificate scmCert,
public static CertPath addRootCertInPath(X509Certificate scmCert,
X509Certificate rootCert) throws CertificateException {
ArrayList<X509Certificate> updatedList = new ArrayList<>();
updatedList.add(scmCert);
Expand All @@ -251,7 +206,7 @@ public CertPath addRootCertInPath(X509Certificate scmCert,
return factory.engineGenerateCertPath(updatedList);
}

public CertPath getRootCertPath(X509Certificate rootCert)
public static CertPath getRootCertPath(X509Certificate rootCert)
throws CertificateException {
ArrayList<X509Certificate> updatedList = new ArrayList<>();
updatedList.add(rootCert);
Expand Down

0 comments on commit 3c97adc

Please sign in to comment.