diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml index 0397a08eb3a..70cce849aec 100644 --- a/hadoop-hdds/framework/pom.xml +++ b/hadoop-hdds/framework/pom.xml @@ -53,11 +53,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.ozone hdds-common - - org.apache.ozone - ozone-tools - 1.5.0-SNAPSHOT - org.apache.ozone hdds-managed-rocksdb diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java index 38df9c06e38..ae0c0f0db84 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java @@ -35,9 +35,6 @@ import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateSignRequest; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneSecurityUtil; -import org.apache.hadoop.ozone.repair.RecoverSCMCertificate; -import org.bouncycastle.cert.X509CertificateHolder; -import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,13 +78,12 @@ public class SCMCertificateClient extends DefaultCertificateClient { private ExecutorService executorService; private boolean isPrimarySCM = false; private Consumer saveCertIdCallback; - private String scmDbPath; @SuppressWarnings("parameternumber") public SCMCertificateClient(SecurityConfig securityConfig, SCMSecurityProtocolClientSideTranslatorPB scmClient, String scmId, String clusterId, String scmCertId, String hostname, - boolean isPrimarySCM, Consumer saveCertId,String scmDbPath) { + boolean isPrimarySCM, Consumer saveCertId) { super(securityConfig, scmClient, LOG, scmCertId, COMPONENT_NAME, HddsUtils.threadNamePrefix(scmId), saveCertId, null); this.scmId = scmId; @@ -95,7 +91,6 @@ public SCMCertificateClient(SecurityConfig securityConfig, this.scmHostname = hostname; this.isPrimarySCM = isPrimarySCM; this.saveCertIdCallback = saveCertId; - this.scmDbPath = scmDbPath; } public SCMCertificateClient(SecurityConfig securityConfig, @@ -134,24 +129,20 @@ public SCMCertificateClient( component); } - public String getScmDbPath() { - return scmDbPath; - } - /** * Returns a CSR builder that can be used to creates a Certificate signing * request. * * @return CertificateSignRequest.Builder */ - public CertificateSignRequest.Builder getCSRBuilder() - throws CertificateException { + public CertificateSignRequest.Builder configureCSRBuilder() + throws SCMSecurityException { String subject = SCM_SUB_CA_PREFIX + scmHostname; LOG.info("Creating csr for SCM->hostName:{},scmId:{},clusterId:{}," + "subject:{}", scmHostname, scmId, cId, subject); - return super.getCSRBuilder() + return super.configureCSRBuilder() .setSubject(subject) .setScmID(scmId) .setClusterID(cId) @@ -286,28 +277,15 @@ public synchronized void close() throws IOException { @Override protected void recoverStateIfNeeded(InitResponse state) throws IOException { LOG.info("Init response: {}", state); - // make this check based on a config? - boolean checkDB = false; switch (state) { case SUCCESS: LOG.info("Initialization successful."); break; case GETCERT: - boolean checkDbSuccess = false; - if (checkDB) { - try { - checkDbSuccess = RecoverSCMCertificate.getAndStoreCerts(getSecurityConfig(), getScmDbPath()); - } catch (Exception e) { - LOG.error("Couldn't retrieve certs from DB"); - throw new IOException(e.getMessage()); - } - } - if (!checkDbSuccess) { - if (!isPrimarySCM) { - getRootCASignedSCMCert(); - } else { - getPrimarySCMSelfSignedCert(); - } + if (!isPrimarySCM) { + getRootCASignedSCMCert(); + } else { + getPrimarySCMSelfSignedCert(); } LOG.info("Successfully stored SCM signed certificate."); break; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java index e6400d80da2..f0d78b23079 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java @@ -30,8 +30,6 @@ import org.apache.hadoop.hdds.security.x509.certificate.authority.profile.PKIProfile; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; -import org.apache.hadoop.hdds.server.ServerUtils; -import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ratis.client.RaftClient; import org.apache.ratis.conf.RaftProperties; @@ -45,11 +43,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; import java.math.BigInteger; import java.net.InetAddress; -import java.nio.file.Paths; import java.security.cert.X509Certificate; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -82,10 +78,7 @@ public static void initializeSecurity(SCMStorageConfig scmStorageConfig, OzoneConfiguration conf, String scmHostname, boolean primaryscm) throws IOException { LOG.info("Initializing secure StorageContainerManager."); - File scmDbDir = ServerUtils.getScmDbDir(conf); - String dbPath = Paths.get(scmDbDir.getAbsolutePath(), OzoneConsts.SCM_DB_NAME) - .toFile().getAbsolutePath(); - LOG.info("SCM DB Path is : {}"); + SecurityConfig securityConfig = new SecurityConfig(conf); SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient = getScmSecurityClientWithFixedDuration(conf); @@ -100,7 +93,7 @@ public static void initializeSecurity(SCMStorageConfig scmStorageConfig, LOG.error("Failed to set new certificate ID", e); throw new RuntimeException("Failed to set new certificate ID"); } - }, dbPath)) { + })) { certClient.initWithRecovery(); } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java index 661d1e4c94b..36b82a21b26 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java @@ -15,9 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.ozone.repair; +package org.apache.hadoop.ozone.debug; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.cli.SubcommandWithParent; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.security.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.authority.CAType; import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; @@ -27,16 +28,19 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.debug.RocksDBUtils; import org.bouncycastle.jcajce.provider.asymmetric.x509.CertificateFactory; +import org.kohsuke.MetaInfServices; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; +import picocli.CommandLine; import java.io.IOException; +import java.io.PrintWriter; import java.math.BigInteger; import java.net.InetAddress; import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; import java.security.cert.CertPath; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; @@ -46,19 +50,92 @@ import java.util.ArrayList; import java.util.Optional; import java.util.Arrays; +import java.util.concurrent.Callable; import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.VALID_SCM_CERTS; import static org.apache.hadoop.hdds.security.x509.certificate.client.DefaultCertificateClient.CERT_FILE_NAME_FORMAT; -import static org.apache.hadoop.hdds.utils.db.DBDefinition.LOG; +import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.removeTrailingSlashIfNeeded; /** * In case of accidental deletion of SCM certificates from local storage, * this tool restores the certs that are persisted into the SCM DB. - * Note that this will only work if the SCM has persisted certs in its RocksDB - * and the private keys are intact and not lost. If private keys of the SCM are - * lost, this tool is not of much use. + * Note that this will only work if the SCM has persisted certs in its RocksDB. */ -public class RecoverSCMCertificate { +@CommandLine.Command( + name = "cert-recover", + description = "Recover Deleted SCM Certificate from RocksDB") +@MetaInfServices(SubcommandWithParent.class) +public class RecoverSCMCertificate implements Callable, SubcommandWithParent { + + @CommandLine.Option(names = {"--db"}, + required = true, + description = "SCM DB Path") + private String dbPath; + + @CommandLine.ParentCommand + private OzoneDebug parent; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public Class getParentType() { + return OzoneDebug.class; + } + + private PrintWriter err() { + return spec.commandLine().getErr(); + } + + private PrintWriter out() { + return spec.commandLine().getOut(); + } + + @Override + public Void call() throws Exception { + dbPath = removeTrailingSlashIfNeeded(dbPath); + String tableName = VALID_SCM_CERTS.getName(); + DBDefinition dbDefinition = + DBDefinitionFactory.getDefinition(Paths.get(dbPath), new OzoneConfiguration()); + if (dbDefinition == null) { + throw new Exception("Error: Incorrect DB Path"); + } + DBColumnFamilyDefinition columnFamilyDefinition = + getDbColumnFamilyDefinition(tableName, dbDefinition); + + try { + List cfDescList = RocksDBUtils.getColumnFamilyDescriptors(dbPath); + final List cfHandleList = new ArrayList<>(); + byte[] tableNameBytes = tableName.getBytes(StandardCharsets.UTF_8); + ColumnFamilyHandle cfHandle = null; + try (ManagedRocksDB db = ManagedRocksDB.openReadOnly(dbPath, cfDescList, + cfHandleList)) { + cfHandle = getColumnFamilyHandle(cfHandleList, tableNameBytes); + + Map allCerts = getAllCerts(columnFamilyDefinition, cfHandle, db); + out().println("All Certs in DB : " + allCerts.keySet()); + String hostName = InetAddress.getLocalHost().getCanonicalHostName(); + out().println("Host: " + hostName); + + X509Certificate subCertificate = getSubCertificate(allCerts, hostName); + X509Certificate rootCertificate = getRootCertificate(allCerts); + + out().println("Sub cert serialID for this host: " + subCertificate.getSerialNumber().toString()); + out().println("Root cert serialID: " + rootCertificate.getSerialNumber().toString()); + + boolean isRootCA = false; + + String caPrincipal = rootCertificate.getSubjectDN().getName(); + if (caPrincipal.contains(hostName)) { + isRootCA = true; + } + storeCerts(subCertificate, rootCertificate, isRootCA); + } + } catch (RocksDBException | CertificateException exception) { + err().print("Failed to recover scm cert"); + } + return null; + } private static ColumnFamilyHandle getColumnFamilyHandle( List cfHandleList, byte[] tableNameBytes) throws Exception { @@ -80,7 +157,10 @@ private static X509Certificate getRootCertificate( Optional cert = allCerts.values().stream().filter( c -> c.getSubjectDN().getName() .contains(OzoneConsts.SCM_ROOT_CA_PREFIX)).findFirst(); - return cert.orElse(null); + if (!cert.isPresent()) { + throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet()); + } + return cert.get(); } @@ -90,53 +170,13 @@ private static X509Certificate getSubCertificate( c -> c.getSubjectDN().getName() .contains(OzoneConsts.SCM_SUB_CA_PREFIX) && c.getSubjectDN() .getName().contains(hostName)).findFirst(); - return cert.orElse(null); - } - - - public static boolean getAndStoreCerts(SecurityConfig conf,String dbPath) - throws Exception { - DBDefinition dbDefinition = new SCMDBDefinition(); - String tableName = VALID_SCM_CERTS.getName(); - DBColumnFamilyDefinition columnFamilyDefinition = - getDbColumnFamilyDefinition(tableName, dbDefinition); - List cfDescList = RocksDBUtils.getColumnFamilyDescriptors(dbPath); - final List cfHandleList = new ArrayList<>(); - byte[] tableNameBytes = tableName.getBytes(StandardCharsets.UTF_8); - ColumnFamilyHandle cfHandle = null; - try (ManagedRocksDB db = ManagedRocksDB.openReadOnly(dbPath, cfDescList, - cfHandleList)) { - cfHandle = getColumnFamilyHandle(cfHandleList, tableNameBytes); - - Map allCerts = getAllCerts(columnFamilyDefinition, cfHandle, db); - LOG.info("All Certs in DB : " + allCerts.keySet()); - String hostName = InetAddress.getLocalHost().getCanonicalHostName(); - LOG.info("Host: " + hostName); - - X509Certificate subCertificate = getSubCertificate(allCerts, hostName); - X509Certificate rootCertificate = getRootCertificate(allCerts); - - boolean containsCerts = subCertificate!=null && rootCertificate!=null; - - if (containsCerts) { - LOG.info("Sub cert serialID for this host: " + subCertificate.getSerialNumber() - .toString()); - LOG.info("Root cert serialID: " + rootCertificate.getSerialNumber().toString()); - - boolean isRootCA = false; - - String caPrincipal = rootCertificate.getSubjectDN().getName(); - if (caPrincipal.contains(hostName)) { - isRootCA = true; - } - storeCerts(subCertificate, rootCertificate, isRootCA, conf); - } - - return containsCerts; + if (!cert.isPresent()) { + throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet()); } + return cert.get(); } - public static Map getAllCerts( + private static Map getAllCerts( DBColumnFamilyDefinition columnFamilyDefinition, ColumnFamilyHandle cfHandle, ManagedRocksDB db) throws IOException, RocksDBException { Map allCerts = new HashMap<>(); @@ -165,13 +205,14 @@ private static DBColumnFamilyDefinition getDbColumnFamilyDefinition( return columnFamilyDefinition; } - private static void storeCerts(X509Certificate scmCertificate, - X509Certificate rootCertificate, boolean isRootCA, SecurityConfig securityConfig) + private void storeCerts(X509Certificate scmCertificate, + X509Certificate rootCertificate, boolean isRootCA) throws CertificateException, IOException { + SecurityConfig securityConfig = new SecurityConfig(parent.getOzoneConf()); CertificateCodec certCodec = new CertificateCodec(securityConfig, SCMCertificateClient.COMPONENT_NAME); - LOG.info("Writing certs to path : " + certCodec.getLocation().toString()); + out().println("Writing certs to path : " + certCodec.getLocation().toString()); CertPath certPath = addRootCertInPath(scmCertificate, rootCertificate); CertPath rootCertPath = getRootCertPath(rootCertificate); @@ -191,13 +232,13 @@ private static void storeCerts(X509Certificate scmCertificate, if (isRootCA) { CertificateCodec rootCertCodec = new CertificateCodec(securityConfig, OzoneConsts.SCM_ROOT_CA_COMPONENT_NAME); - LOG.info("Writing root certs to path : " + rootCertCodec.getLocation().toString()); + out().println("Writing root certs to path : " + rootCertCodec.getLocation().toString()); rootCertCodec.writeCertificate(rootCertCodec.getLocation().toAbsolutePath(), securityConfig.getCertificateFileName(), encodedRootCert); } } - public static CertPath addRootCertInPath(X509Certificate scmCert, + public CertPath addRootCertInPath(X509Certificate scmCert, X509Certificate rootCert) throws CertificateException { ArrayList updatedList = new ArrayList<>(); updatedList.add(scmCert); @@ -206,7 +247,7 @@ public static CertPath addRootCertInPath(X509Certificate scmCert, return factory.engineGenerateCertPath(updatedList); } - public static CertPath getRootCertPath(X509Certificate rootCert) + public CertPath getRootCertPath(X509Certificate rootCert) throws CertificateException { ArrayList updatedList = new ArrayList<>(); updatedList.add(rootCert);