From 4e92b68f91c624d0f7af4c11558ec28fb52f5a27 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Thu, 6 Jun 2024 15:54:27 +0530 Subject: [PATCH 1/7] HDDS-10984. Tool to restore SCM certificates from RocksDB. --- .../ozone/debug/RecoverSCMCertificate.java | 257 ++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java new file mode 100644 index 00000000000..cde9e350399 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.debug; + +import org.apache.hadoop.hdds.cli.SubcommandWithParent; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.security.SecurityConfig; +import org.apache.hadoop.hdds.security.x509.certificate.authority.CAType; +import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; +import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; +import org.apache.hadoop.hdds.utils.db.DBDefinition; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; +import org.apache.hadoop.ozone.OzoneConsts; +import org.bouncycastle.jcajce.provider.asymmetric.x509.CertificateFactory; +import org.kohsuke.MetaInfServices; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.RocksDBException; +import picocli.CommandLine; + +import java.io.IOException; +import java.io.PrintWriter; +import java.math.BigInteger; +import java.net.InetAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.security.cert.CertPath; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.Map; +import java.util.HashMap; +import java.util.List; +import java.util.ArrayList; +import java.util.Optional; +import java.util.Arrays; +import java.util.concurrent.Callable; + +import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.VALID_SCM_CERTS; +import static org.apache.hadoop.hdds.security.x509.certificate.client.DefaultCertificateClient.CERT_FILE_NAME_FORMAT; +import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.removeTrailingSlashIfNeeded; + +/** + * In case of accidental deletion of SCM certificates from local storage, + * this tool restores the certs that are persisted into the SCM DB. + * Note that this will only work if the SCM has persisted certs in its RocksDB. + */ +@CommandLine.Command( + name = "cert-recover", + description = "Recover Deleted SCM Certificate from RocksDB") +@MetaInfServices(SubcommandWithParent.class) +public class RecoverSCMCertificate implements Callable, SubcommandWithParent{ + + @CommandLine.Option(names = {"--db"}, + required = true, + description = "SCM DB Path") + private String dbPath; + + @CommandLine.ParentCommand + private OzoneDebug parent; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public Class getParentType() { + return OzoneDebug.class; + } + + private PrintWriter err() { + return spec.commandLine().getErr(); + } + + private PrintWriter out() { + return spec.commandLine().getOut(); + } + + @Override + public Void call() throws Exception { + dbPath = removeTrailingSlashIfNeeded(dbPath); + String tableName = VALID_SCM_CERTS.getName(); + DBDefinition dbDefinition = + DBDefinitionFactory.getDefinition(Paths.get(dbPath), new OzoneConfiguration()); + if (dbDefinition == null) { + throw new Exception("Error: Incorrect DB Path"); + } + DBColumnFamilyDefinition columnFamilyDefinition = + getDbColumnFamilyDefinition(tableName, dbDefinition); + + try { + List cfDescList = RocksDBUtils.getColumnFamilyDescriptors(dbPath); + final List cfHandleList = new ArrayList<>(); + byte[] tableNameBytes = tableName.getBytes(StandardCharsets.UTF_8); + ColumnFamilyHandle cfHandle = null; + try (ManagedRocksDB db = ManagedRocksDB.openReadOnly(dbPath, cfDescList, + cfHandleList)) { + cfHandle = getColumnFamilyHandle(cfHandleList, tableNameBytes); + + Map allCerts = getAllCerts(columnFamilyDefinition, cfHandle, db); + out().println("All Certs in DB : " + allCerts.keySet()); + String hostName = InetAddress.getLocalHost().getCanonicalHostName(); + out().println("Host: " + hostName); + + X509Certificate subCertificate = getSubCertificate(allCerts, hostName); + X509Certificate rootCertificate = getRootCertificate(allCerts); + + out().println("Sub cert serialID for this host: " + subCertificate.getSerialNumber().toString()); + out().println("Root cert serialID: " + rootCertificate.getSerialNumber().toString()); + + boolean isRootCA = false; + + String caPrincipal = rootCertificate.getSubjectDN().getName(); + if (caPrincipal.contains(hostName)) { + isRootCA = true; + } + storeCerts(subCertificate, rootCertificate, isRootCA); + } + } catch (RocksDBException | CertificateException exception) { + err().print("Failed to recover scm cert"); + } + return null; + } + + private static ColumnFamilyHandle getColumnFamilyHandle( + List cfHandleList, byte[] tableNameBytes) throws Exception { + ColumnFamilyHandle cfHandle = null; + for (ColumnFamilyHandle cf : cfHandleList) { + if (Arrays.equals(cf.getName(), tableNameBytes)) { + cfHandle = cf; + break; + } + } + if (cfHandle == null) { + throw new Exception("Error: VALID_SCM_CERTS table not found in DB"); + } + return cfHandle; + } + + private static X509Certificate getRootCertificate( + Map allCerts) throws Exception { + Optional cert = allCerts.values().stream().filter( + c -> c.getSubjectDN().getName() + .contains(OzoneConsts.SCM_ROOT_CA_PREFIX)).findFirst(); + if (!cert.isPresent()) { + throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet()); + } + return cert.get(); + } + + + private static X509Certificate getSubCertificate( + Map allCerts, String hostName) throws Exception { + Optional cert = allCerts.values().stream().filter( + c -> c.getSubjectDN().getName() + .contains(OzoneConsts.SCM_SUB_CA_PREFIX) && c.getSubjectDN() + .getName().contains(hostName)).findFirst(); + if (!cert.isPresent()) { + throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet()); + } + return cert.get(); + } + + private static Map getAllCerts( + DBColumnFamilyDefinition columnFamilyDefinition, + ColumnFamilyHandle cfHandle, ManagedRocksDB db) throws IOException, RocksDBException { + Map allCerts = new HashMap<>(); + ManagedRocksIterator rocksIterator = ManagedRocksIterator.managed(db.get().newIterator(cfHandle)); + rocksIterator.get().seekToFirst(); + while (rocksIterator.get().isValid()) { + BigInteger id = (BigInteger) columnFamilyDefinition.getKeyCodec() + .fromPersistedFormat(rocksIterator.get().key()); + X509Certificate certificate = + (X509Certificate) columnFamilyDefinition.getValueCodec() + .fromPersistedFormat(rocksIterator.get().value()); + allCerts.put(id, certificate); + rocksIterator.get().next(); + } + return allCerts; + } + + private static DBColumnFamilyDefinition getDbColumnFamilyDefinition( + String tableName, DBDefinition dbDefinition) throws Exception { + DBColumnFamilyDefinition columnFamilyDefinition = + dbDefinition.getColumnFamily(tableName); + if (columnFamilyDefinition == null) { + throw new Exception( + "Error: VALID_SCM_CERTS table no found in Definition"); + } + return columnFamilyDefinition; + } + + private void storeCerts(X509Certificate scmCertificate, + X509Certificate rootCertificate, boolean isRootCA) + throws CertificateException, IOException { + SecurityConfig securityConfig = new SecurityConfig(parent.getOzoneConf()); + CertificateCodec certCodec = + new CertificateCodec(securityConfig, SCMCertificateClient.COMPONENT_NAME); + + out().println("Writing certs to path : " + certCodec.getLocation().toString()); + + CertPath certPath = addRootCertInPath(scmCertificate, rootCertificate); + CertPath rootCertPath = getRootCertPath(rootCertificate); + String encodedCert = CertificateCodec.getPEMEncodedString(certPath); + String certName = String.format(CERT_FILE_NAME_FORMAT, + CAType.NONE.getFileNamePrefix() + scmCertificate.getSerialNumber().toString()); + certCodec.writeCertificate(certName, encodedCert); + + String rootCertName = String.format(CERT_FILE_NAME_FORMAT, + CAType.SUBORDINATE.getFileNamePrefix() + rootCertificate.getSerialNumber().toString()); + String encodedRootCert = CertificateCodec.getPEMEncodedString(rootCertPath); + certCodec.writeCertificate(rootCertName, encodedRootCert); + + certCodec.writeCertificate(certCodec.getLocation().toAbsolutePath(), + securityConfig.getCertificateFileName(), encodedCert); + + if (isRootCA) { + CertificateCodec rootCertCodec = + new CertificateCodec(securityConfig, OzoneConsts.SCM_ROOT_CA_COMPONENT_NAME); + out().println("Writing root certs to path : " + rootCertCodec.getLocation().toString()); + rootCertCodec.writeCertificate(rootCertCodec.getLocation().toAbsolutePath(), + securityConfig.getCertificateFileName(), encodedRootCert); + } + } + + public CertPath addRootCertInPath(X509Certificate scmCert, + X509Certificate rootCert) throws CertificateException { + ArrayList updatedList = new ArrayList<>(); + updatedList.add(scmCert); + updatedList.add(rootCert); + CertificateFactory factory = new CertificateFactory(); + return factory.engineGenerateCertPath(updatedList); + } + + public CertPath getRootCertPath(X509Certificate rootCert) + throws CertificateException { + ArrayList updatedList = new ArrayList<>(); + updatedList.add(rootCert); + CertificateFactory factory = new CertificateFactory(); + return factory.engineGenerateCertPath(updatedList); + } + } From 18a1afca323979f4483536b1c04a986a058b14db Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Thu, 6 Jun 2024 16:02:43 +0530 Subject: [PATCH 2/7] fix checkstyle (cherry picked from commit 6b5b7773734307cd85efdcbe01a023d863141a95) --- .../apache/hadoop/ozone/debug/RecoverSCMCertificate.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java index cde9e350399..36b82a21b26 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java @@ -65,7 +65,7 @@ name = "cert-recover", description = "Recover Deleted SCM Certificate from RocksDB") @MetaInfServices(SubcommandWithParent.class) -public class RecoverSCMCertificate implements Callable, SubcommandWithParent{ +public class RecoverSCMCertificate implements Callable, SubcommandWithParent { @CommandLine.Option(names = {"--db"}, required = true, @@ -135,7 +135,7 @@ public Void call() throws Exception { err().print("Failed to recover scm cert"); } return null; - } + } private static ColumnFamilyHandle getColumnFamilyHandle( List cfHandleList, byte[] tableNameBytes) throws Exception { @@ -254,4 +254,4 @@ public CertPath getRootCertPath(X509Certificate rootCert) CertificateFactory factory = new CertificateFactory(); return factory.engineGenerateCertPath(updatedList); } - } +} From 166883a73a479bcdd361b1505525a5de52d5751f Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Thu, 11 Jul 2024 14:17:05 +0530 Subject: [PATCH 3/7] automatic recovery in cert client code draft (cherry picked from commit 3c97adcdb5a23530c6ccde10da813eda89b88d03) --- hadoop-hdds/framework/pom.xml | 5 + .../client/SCMCertificateClient.java | 38 ++++- .../hadoop/hdds/scm/ha/HASecurityUtils.java | 11 +- .../ozone/debug/RecoverSCMCertificate.java | 159 +++++++----------- 4 files changed, 103 insertions(+), 110 deletions(-) diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml index 70cce849aec..0397a08eb3a 100644 --- a/hadoop-hdds/framework/pom.xml +++ b/hadoop-hdds/framework/pom.xml @@ -53,6 +53,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.ozone hdds-common + + org.apache.ozone + ozone-tools + 1.5.0-SNAPSHOT + org.apache.ozone hdds-managed-rocksdb diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java index ae0c0f0db84..38df9c06e38 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java @@ -35,6 +35,9 @@ import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateSignRequest; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneSecurityUtil; +import org.apache.hadoop.ozone.repair.RecoverSCMCertificate; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,12 +81,13 @@ public class SCMCertificateClient extends DefaultCertificateClient { private ExecutorService executorService; private boolean isPrimarySCM = false; private Consumer saveCertIdCallback; + private String scmDbPath; @SuppressWarnings("parameternumber") public SCMCertificateClient(SecurityConfig securityConfig, SCMSecurityProtocolClientSideTranslatorPB scmClient, String scmId, String clusterId, String scmCertId, String hostname, - boolean isPrimarySCM, Consumer saveCertId) { + boolean isPrimarySCM, Consumer saveCertId,String scmDbPath) { super(securityConfig, scmClient, LOG, scmCertId, COMPONENT_NAME, HddsUtils.threadNamePrefix(scmId), saveCertId, null); this.scmId = scmId; @@ -91,6 +95,7 @@ public SCMCertificateClient(SecurityConfig securityConfig, this.scmHostname = hostname; this.isPrimarySCM = isPrimarySCM; this.saveCertIdCallback = saveCertId; + this.scmDbPath = scmDbPath; } public SCMCertificateClient(SecurityConfig securityConfig, @@ -129,20 +134,24 @@ public SCMCertificateClient( component); } + public String getScmDbPath() { + return scmDbPath; + } + /** * Returns a CSR builder that can be used to creates a Certificate signing * request. * * @return CertificateSignRequest.Builder */ - public CertificateSignRequest.Builder configureCSRBuilder() - throws SCMSecurityException { + public CertificateSignRequest.Builder getCSRBuilder() + throws CertificateException { String subject = SCM_SUB_CA_PREFIX + scmHostname; LOG.info("Creating csr for SCM->hostName:{},scmId:{},clusterId:{}," + "subject:{}", scmHostname, scmId, cId, subject); - return super.configureCSRBuilder() + return super.getCSRBuilder() .setSubject(subject) .setScmID(scmId) .setClusterID(cId) @@ -277,15 +286,28 @@ public synchronized void close() throws IOException { @Override protected void recoverStateIfNeeded(InitResponse state) throws IOException { LOG.info("Init response: {}", state); + // make this check based on a config? + boolean checkDB = false; switch (state) { case SUCCESS: LOG.info("Initialization successful."); break; case GETCERT: - if (!isPrimarySCM) { - getRootCASignedSCMCert(); - } else { - getPrimarySCMSelfSignedCert(); + boolean checkDbSuccess = false; + if (checkDB) { + try { + checkDbSuccess = RecoverSCMCertificate.getAndStoreCerts(getSecurityConfig(), getScmDbPath()); + } catch (Exception e) { + LOG.error("Couldn't retrieve certs from DB"); + throw new IOException(e.getMessage()); + } + } + if (!checkDbSuccess) { + if (!isPrimarySCM) { + getRootCASignedSCMCert(); + } else { + getPrimarySCMSelfSignedCert(); + } } LOG.info("Successfully stored SCM signed certificate."); break; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java index f0d78b23079..e6400d80da2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java @@ -30,6 +30,8 @@ import org.apache.hadoop.hdds.security.x509.certificate.authority.profile.PKIProfile; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; +import org.apache.hadoop.hdds.server.ServerUtils; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ratis.client.RaftClient; import org.apache.ratis.conf.RaftProperties; @@ -43,9 +45,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.math.BigInteger; import java.net.InetAddress; +import java.nio.file.Paths; import java.security.cert.X509Certificate; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -78,7 +82,10 @@ public static void initializeSecurity(SCMStorageConfig scmStorageConfig, OzoneConfiguration conf, String scmHostname, boolean primaryscm) throws IOException { LOG.info("Initializing secure StorageContainerManager."); - + File scmDbDir = ServerUtils.getScmDbDir(conf); + String dbPath = Paths.get(scmDbDir.getAbsolutePath(), OzoneConsts.SCM_DB_NAME) + .toFile().getAbsolutePath(); + LOG.info("SCM DB Path is : {}"); SecurityConfig securityConfig = new SecurityConfig(conf); SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient = getScmSecurityClientWithFixedDuration(conf); @@ -93,7 +100,7 @@ public static void initializeSecurity(SCMStorageConfig scmStorageConfig, LOG.error("Failed to set new certificate ID", e); throw new RuntimeException("Failed to set new certificate ID"); } - })) { + }, dbPath)) { certClient.initWithRecovery(); } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java index 36b82a21b26..661d1e4c94b 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java @@ -15,10 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.ozone.debug; +package org.apache.hadoop.ozone.repair; -import org.apache.hadoop.hdds.cli.SubcommandWithParent; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; import org.apache.hadoop.hdds.security.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.authority.CAType; import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; @@ -28,19 +27,16 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.debug.RocksDBUtils; import org.bouncycastle.jcajce.provider.asymmetric.x509.CertificateFactory; -import org.kohsuke.MetaInfServices; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; -import picocli.CommandLine; import java.io.IOException; -import java.io.PrintWriter; import java.math.BigInteger; import java.net.InetAddress; import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; import java.security.cert.CertPath; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; @@ -50,92 +46,19 @@ import java.util.ArrayList; import java.util.Optional; import java.util.Arrays; -import java.util.concurrent.Callable; import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.VALID_SCM_CERTS; import static org.apache.hadoop.hdds.security.x509.certificate.client.DefaultCertificateClient.CERT_FILE_NAME_FORMAT; -import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.removeTrailingSlashIfNeeded; +import static org.apache.hadoop.hdds.utils.db.DBDefinition.LOG; /** * In case of accidental deletion of SCM certificates from local storage, * this tool restores the certs that are persisted into the SCM DB. - * Note that this will only work if the SCM has persisted certs in its RocksDB. + * Note that this will only work if the SCM has persisted certs in its RocksDB + * and the private keys are intact and not lost. If private keys of the SCM are + * lost, this tool is not of much use. */ -@CommandLine.Command( - name = "cert-recover", - description = "Recover Deleted SCM Certificate from RocksDB") -@MetaInfServices(SubcommandWithParent.class) -public class RecoverSCMCertificate implements Callable, SubcommandWithParent { - - @CommandLine.Option(names = {"--db"}, - required = true, - description = "SCM DB Path") - private String dbPath; - - @CommandLine.ParentCommand - private OzoneDebug parent; - - @CommandLine.Spec - private CommandLine.Model.CommandSpec spec; - - @Override - public Class getParentType() { - return OzoneDebug.class; - } - - private PrintWriter err() { - return spec.commandLine().getErr(); - } - - private PrintWriter out() { - return spec.commandLine().getOut(); - } - - @Override - public Void call() throws Exception { - dbPath = removeTrailingSlashIfNeeded(dbPath); - String tableName = VALID_SCM_CERTS.getName(); - DBDefinition dbDefinition = - DBDefinitionFactory.getDefinition(Paths.get(dbPath), new OzoneConfiguration()); - if (dbDefinition == null) { - throw new Exception("Error: Incorrect DB Path"); - } - DBColumnFamilyDefinition columnFamilyDefinition = - getDbColumnFamilyDefinition(tableName, dbDefinition); - - try { - List cfDescList = RocksDBUtils.getColumnFamilyDescriptors(dbPath); - final List cfHandleList = new ArrayList<>(); - byte[] tableNameBytes = tableName.getBytes(StandardCharsets.UTF_8); - ColumnFamilyHandle cfHandle = null; - try (ManagedRocksDB db = ManagedRocksDB.openReadOnly(dbPath, cfDescList, - cfHandleList)) { - cfHandle = getColumnFamilyHandle(cfHandleList, tableNameBytes); - - Map allCerts = getAllCerts(columnFamilyDefinition, cfHandle, db); - out().println("All Certs in DB : " + allCerts.keySet()); - String hostName = InetAddress.getLocalHost().getCanonicalHostName(); - out().println("Host: " + hostName); - - X509Certificate subCertificate = getSubCertificate(allCerts, hostName); - X509Certificate rootCertificate = getRootCertificate(allCerts); - - out().println("Sub cert serialID for this host: " + subCertificate.getSerialNumber().toString()); - out().println("Root cert serialID: " + rootCertificate.getSerialNumber().toString()); - - boolean isRootCA = false; - - String caPrincipal = rootCertificate.getSubjectDN().getName(); - if (caPrincipal.contains(hostName)) { - isRootCA = true; - } - storeCerts(subCertificate, rootCertificate, isRootCA); - } - } catch (RocksDBException | CertificateException exception) { - err().print("Failed to recover scm cert"); - } - return null; - } +public class RecoverSCMCertificate { private static ColumnFamilyHandle getColumnFamilyHandle( List cfHandleList, byte[] tableNameBytes) throws Exception { @@ -157,10 +80,7 @@ private static X509Certificate getRootCertificate( Optional cert = allCerts.values().stream().filter( c -> c.getSubjectDN().getName() .contains(OzoneConsts.SCM_ROOT_CA_PREFIX)).findFirst(); - if (!cert.isPresent()) { - throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet()); - } - return cert.get(); + return cert.orElse(null); } @@ -170,13 +90,53 @@ private static X509Certificate getSubCertificate( c -> c.getSubjectDN().getName() .contains(OzoneConsts.SCM_SUB_CA_PREFIX) && c.getSubjectDN() .getName().contains(hostName)).findFirst(); - if (!cert.isPresent()) { - throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet()); + return cert.orElse(null); + } + + + public static boolean getAndStoreCerts(SecurityConfig conf,String dbPath) + throws Exception { + DBDefinition dbDefinition = new SCMDBDefinition(); + String tableName = VALID_SCM_CERTS.getName(); + DBColumnFamilyDefinition columnFamilyDefinition = + getDbColumnFamilyDefinition(tableName, dbDefinition); + List cfDescList = RocksDBUtils.getColumnFamilyDescriptors(dbPath); + final List cfHandleList = new ArrayList<>(); + byte[] tableNameBytes = tableName.getBytes(StandardCharsets.UTF_8); + ColumnFamilyHandle cfHandle = null; + try (ManagedRocksDB db = ManagedRocksDB.openReadOnly(dbPath, cfDescList, + cfHandleList)) { + cfHandle = getColumnFamilyHandle(cfHandleList, tableNameBytes); + + Map allCerts = getAllCerts(columnFamilyDefinition, cfHandle, db); + LOG.info("All Certs in DB : " + allCerts.keySet()); + String hostName = InetAddress.getLocalHost().getCanonicalHostName(); + LOG.info("Host: " + hostName); + + X509Certificate subCertificate = getSubCertificate(allCerts, hostName); + X509Certificate rootCertificate = getRootCertificate(allCerts); + + boolean containsCerts = subCertificate!=null && rootCertificate!=null; + + if (containsCerts) { + LOG.info("Sub cert serialID for this host: " + subCertificate.getSerialNumber() + .toString()); + LOG.info("Root cert serialID: " + rootCertificate.getSerialNumber().toString()); + + boolean isRootCA = false; + + String caPrincipal = rootCertificate.getSubjectDN().getName(); + if (caPrincipal.contains(hostName)) { + isRootCA = true; + } + storeCerts(subCertificate, rootCertificate, isRootCA, conf); + } + + return containsCerts; } - return cert.get(); } - private static Map getAllCerts( + public static Map getAllCerts( DBColumnFamilyDefinition columnFamilyDefinition, ColumnFamilyHandle cfHandle, ManagedRocksDB db) throws IOException, RocksDBException { Map allCerts = new HashMap<>(); @@ -205,14 +165,13 @@ private static DBColumnFamilyDefinition getDbColumnFamilyDefinition( return columnFamilyDefinition; } - private void storeCerts(X509Certificate scmCertificate, - X509Certificate rootCertificate, boolean isRootCA) + private static void storeCerts(X509Certificate scmCertificate, + X509Certificate rootCertificate, boolean isRootCA, SecurityConfig securityConfig) throws CertificateException, IOException { - SecurityConfig securityConfig = new SecurityConfig(parent.getOzoneConf()); CertificateCodec certCodec = new CertificateCodec(securityConfig, SCMCertificateClient.COMPONENT_NAME); - out().println("Writing certs to path : " + certCodec.getLocation().toString()); + LOG.info("Writing certs to path : " + certCodec.getLocation().toString()); CertPath certPath = addRootCertInPath(scmCertificate, rootCertificate); CertPath rootCertPath = getRootCertPath(rootCertificate); @@ -232,13 +191,13 @@ private void storeCerts(X509Certificate scmCertificate, if (isRootCA) { CertificateCodec rootCertCodec = new CertificateCodec(securityConfig, OzoneConsts.SCM_ROOT_CA_COMPONENT_NAME); - out().println("Writing root certs to path : " + rootCertCodec.getLocation().toString()); + LOG.info("Writing root certs to path : " + rootCertCodec.getLocation().toString()); rootCertCodec.writeCertificate(rootCertCodec.getLocation().toAbsolutePath(), securityConfig.getCertificateFileName(), encodedRootCert); } } - public CertPath addRootCertInPath(X509Certificate scmCert, + public static CertPath addRootCertInPath(X509Certificate scmCert, X509Certificate rootCert) throws CertificateException { ArrayList updatedList = new ArrayList<>(); updatedList.add(scmCert); @@ -247,7 +206,7 @@ public CertPath addRootCertInPath(X509Certificate scmCert, return factory.engineGenerateCertPath(updatedList); } - public CertPath getRootCertPath(X509Certificate rootCert) + public static CertPath getRootCertPath(X509Certificate rootCert) throws CertificateException { ArrayList updatedList = new ArrayList<>(); updatedList.add(rootCert); From 40ffb3aab95a50e4f48b336b30bce94b8fe5bc3a Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Fri, 30 Aug 2024 01:10:25 +0530 Subject: [PATCH 4/7] Revert "automatic recovery in cert client code draft" This reverts commit 166883a73a479bcdd361b1505525a5de52d5751f. --- hadoop-hdds/framework/pom.xml | 5 - .../client/SCMCertificateClient.java | 38 +---- .../hadoop/hdds/scm/ha/HASecurityUtils.java | 11 +- .../ozone/debug/RecoverSCMCertificate.java | 159 +++++++++++------- 4 files changed, 110 insertions(+), 103 deletions(-) diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml index 0397a08eb3a..70cce849aec 100644 --- a/hadoop-hdds/framework/pom.xml +++ b/hadoop-hdds/framework/pom.xml @@ -53,11 +53,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.ozone hdds-common - - org.apache.ozone - ozone-tools - 1.5.0-SNAPSHOT - org.apache.ozone hdds-managed-rocksdb diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java index 38df9c06e38..ae0c0f0db84 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java @@ -35,9 +35,6 @@ import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateSignRequest; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneSecurityUtil; -import org.apache.hadoop.ozone.repair.RecoverSCMCertificate; -import org.bouncycastle.cert.X509CertificateHolder; -import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,13 +78,12 @@ public class SCMCertificateClient extends DefaultCertificateClient { private ExecutorService executorService; private boolean isPrimarySCM = false; private Consumer saveCertIdCallback; - private String scmDbPath; @SuppressWarnings("parameternumber") public SCMCertificateClient(SecurityConfig securityConfig, SCMSecurityProtocolClientSideTranslatorPB scmClient, String scmId, String clusterId, String scmCertId, String hostname, - boolean isPrimarySCM, Consumer saveCertId,String scmDbPath) { + boolean isPrimarySCM, Consumer saveCertId) { super(securityConfig, scmClient, LOG, scmCertId, COMPONENT_NAME, HddsUtils.threadNamePrefix(scmId), saveCertId, null); this.scmId = scmId; @@ -95,7 +91,6 @@ public SCMCertificateClient(SecurityConfig securityConfig, this.scmHostname = hostname; this.isPrimarySCM = isPrimarySCM; this.saveCertIdCallback = saveCertId; - this.scmDbPath = scmDbPath; } public SCMCertificateClient(SecurityConfig securityConfig, @@ -134,24 +129,20 @@ public SCMCertificateClient( component); } - public String getScmDbPath() { - return scmDbPath; - } - /** * Returns a CSR builder that can be used to creates a Certificate signing * request. * * @return CertificateSignRequest.Builder */ - public CertificateSignRequest.Builder getCSRBuilder() - throws CertificateException { + public CertificateSignRequest.Builder configureCSRBuilder() + throws SCMSecurityException { String subject = SCM_SUB_CA_PREFIX + scmHostname; LOG.info("Creating csr for SCM->hostName:{},scmId:{},clusterId:{}," + "subject:{}", scmHostname, scmId, cId, subject); - return super.getCSRBuilder() + return super.configureCSRBuilder() .setSubject(subject) .setScmID(scmId) .setClusterID(cId) @@ -286,28 +277,15 @@ public synchronized void close() throws IOException { @Override protected void recoverStateIfNeeded(InitResponse state) throws IOException { LOG.info("Init response: {}", state); - // make this check based on a config? - boolean checkDB = false; switch (state) { case SUCCESS: LOG.info("Initialization successful."); break; case GETCERT: - boolean checkDbSuccess = false; - if (checkDB) { - try { - checkDbSuccess = RecoverSCMCertificate.getAndStoreCerts(getSecurityConfig(), getScmDbPath()); - } catch (Exception e) { - LOG.error("Couldn't retrieve certs from DB"); - throw new IOException(e.getMessage()); - } - } - if (!checkDbSuccess) { - if (!isPrimarySCM) { - getRootCASignedSCMCert(); - } else { - getPrimarySCMSelfSignedCert(); - } + if (!isPrimarySCM) { + getRootCASignedSCMCert(); + } else { + getPrimarySCMSelfSignedCert(); } LOG.info("Successfully stored SCM signed certificate."); break; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java index e6400d80da2..f0d78b23079 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java @@ -30,8 +30,6 @@ import org.apache.hadoop.hdds.security.x509.certificate.authority.profile.PKIProfile; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; -import org.apache.hadoop.hdds.server.ServerUtils; -import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ratis.client.RaftClient; import org.apache.ratis.conf.RaftProperties; @@ -45,11 +43,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; import java.math.BigInteger; import java.net.InetAddress; -import java.nio.file.Paths; import java.security.cert.X509Certificate; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -82,10 +78,7 @@ public static void initializeSecurity(SCMStorageConfig scmStorageConfig, OzoneConfiguration conf, String scmHostname, boolean primaryscm) throws IOException { LOG.info("Initializing secure StorageContainerManager."); - File scmDbDir = ServerUtils.getScmDbDir(conf); - String dbPath = Paths.get(scmDbDir.getAbsolutePath(), OzoneConsts.SCM_DB_NAME) - .toFile().getAbsolutePath(); - LOG.info("SCM DB Path is : {}"); + SecurityConfig securityConfig = new SecurityConfig(conf); SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient = getScmSecurityClientWithFixedDuration(conf); @@ -100,7 +93,7 @@ public static void initializeSecurity(SCMStorageConfig scmStorageConfig, LOG.error("Failed to set new certificate ID", e); throw new RuntimeException("Failed to set new certificate ID"); } - }, dbPath)) { + })) { certClient.initWithRecovery(); } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java index 661d1e4c94b..36b82a21b26 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java @@ -15,9 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.ozone.repair; +package org.apache.hadoop.ozone.debug; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.cli.SubcommandWithParent; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.security.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.authority.CAType; import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; @@ -27,16 +28,19 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.debug.RocksDBUtils; import org.bouncycastle.jcajce.provider.asymmetric.x509.CertificateFactory; +import org.kohsuke.MetaInfServices; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; +import picocli.CommandLine; import java.io.IOException; +import java.io.PrintWriter; import java.math.BigInteger; import java.net.InetAddress; import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; import java.security.cert.CertPath; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; @@ -46,19 +50,92 @@ import java.util.ArrayList; import java.util.Optional; import java.util.Arrays; +import java.util.concurrent.Callable; import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.VALID_SCM_CERTS; import static org.apache.hadoop.hdds.security.x509.certificate.client.DefaultCertificateClient.CERT_FILE_NAME_FORMAT; -import static org.apache.hadoop.hdds.utils.db.DBDefinition.LOG; +import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.removeTrailingSlashIfNeeded; /** * In case of accidental deletion of SCM certificates from local storage, * this tool restores the certs that are persisted into the SCM DB. - * Note that this will only work if the SCM has persisted certs in its RocksDB - * and the private keys are intact and not lost. If private keys of the SCM are - * lost, this tool is not of much use. + * Note that this will only work if the SCM has persisted certs in its RocksDB. */ -public class RecoverSCMCertificate { +@CommandLine.Command( + name = "cert-recover", + description = "Recover Deleted SCM Certificate from RocksDB") +@MetaInfServices(SubcommandWithParent.class) +public class RecoverSCMCertificate implements Callable, SubcommandWithParent { + + @CommandLine.Option(names = {"--db"}, + required = true, + description = "SCM DB Path") + private String dbPath; + + @CommandLine.ParentCommand + private OzoneDebug parent; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public Class getParentType() { + return OzoneDebug.class; + } + + private PrintWriter err() { + return spec.commandLine().getErr(); + } + + private PrintWriter out() { + return spec.commandLine().getOut(); + } + + @Override + public Void call() throws Exception { + dbPath = removeTrailingSlashIfNeeded(dbPath); + String tableName = VALID_SCM_CERTS.getName(); + DBDefinition dbDefinition = + DBDefinitionFactory.getDefinition(Paths.get(dbPath), new OzoneConfiguration()); + if (dbDefinition == null) { + throw new Exception("Error: Incorrect DB Path"); + } + DBColumnFamilyDefinition columnFamilyDefinition = + getDbColumnFamilyDefinition(tableName, dbDefinition); + + try { + List cfDescList = RocksDBUtils.getColumnFamilyDescriptors(dbPath); + final List cfHandleList = new ArrayList<>(); + byte[] tableNameBytes = tableName.getBytes(StandardCharsets.UTF_8); + ColumnFamilyHandle cfHandle = null; + try (ManagedRocksDB db = ManagedRocksDB.openReadOnly(dbPath, cfDescList, + cfHandleList)) { + cfHandle = getColumnFamilyHandle(cfHandleList, tableNameBytes); + + Map allCerts = getAllCerts(columnFamilyDefinition, cfHandle, db); + out().println("All Certs in DB : " + allCerts.keySet()); + String hostName = InetAddress.getLocalHost().getCanonicalHostName(); + out().println("Host: " + hostName); + + X509Certificate subCertificate = getSubCertificate(allCerts, hostName); + X509Certificate rootCertificate = getRootCertificate(allCerts); + + out().println("Sub cert serialID for this host: " + subCertificate.getSerialNumber().toString()); + out().println("Root cert serialID: " + rootCertificate.getSerialNumber().toString()); + + boolean isRootCA = false; + + String caPrincipal = rootCertificate.getSubjectDN().getName(); + if (caPrincipal.contains(hostName)) { + isRootCA = true; + } + storeCerts(subCertificate, rootCertificate, isRootCA); + } + } catch (RocksDBException | CertificateException exception) { + err().print("Failed to recover scm cert"); + } + return null; + } private static ColumnFamilyHandle getColumnFamilyHandle( List cfHandleList, byte[] tableNameBytes) throws Exception { @@ -80,7 +157,10 @@ private static X509Certificate getRootCertificate( Optional cert = allCerts.values().stream().filter( c -> c.getSubjectDN().getName() .contains(OzoneConsts.SCM_ROOT_CA_PREFIX)).findFirst(); - return cert.orElse(null); + if (!cert.isPresent()) { + throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet()); + } + return cert.get(); } @@ -90,53 +170,13 @@ private static X509Certificate getSubCertificate( c -> c.getSubjectDN().getName() .contains(OzoneConsts.SCM_SUB_CA_PREFIX) && c.getSubjectDN() .getName().contains(hostName)).findFirst(); - return cert.orElse(null); - } - - - public static boolean getAndStoreCerts(SecurityConfig conf,String dbPath) - throws Exception { - DBDefinition dbDefinition = new SCMDBDefinition(); - String tableName = VALID_SCM_CERTS.getName(); - DBColumnFamilyDefinition columnFamilyDefinition = - getDbColumnFamilyDefinition(tableName, dbDefinition); - List cfDescList = RocksDBUtils.getColumnFamilyDescriptors(dbPath); - final List cfHandleList = new ArrayList<>(); - byte[] tableNameBytes = tableName.getBytes(StandardCharsets.UTF_8); - ColumnFamilyHandle cfHandle = null; - try (ManagedRocksDB db = ManagedRocksDB.openReadOnly(dbPath, cfDescList, - cfHandleList)) { - cfHandle = getColumnFamilyHandle(cfHandleList, tableNameBytes); - - Map allCerts = getAllCerts(columnFamilyDefinition, cfHandle, db); - LOG.info("All Certs in DB : " + allCerts.keySet()); - String hostName = InetAddress.getLocalHost().getCanonicalHostName(); - LOG.info("Host: " + hostName); - - X509Certificate subCertificate = getSubCertificate(allCerts, hostName); - X509Certificate rootCertificate = getRootCertificate(allCerts); - - boolean containsCerts = subCertificate!=null && rootCertificate!=null; - - if (containsCerts) { - LOG.info("Sub cert serialID for this host: " + subCertificate.getSerialNumber() - .toString()); - LOG.info("Root cert serialID: " + rootCertificate.getSerialNumber().toString()); - - boolean isRootCA = false; - - String caPrincipal = rootCertificate.getSubjectDN().getName(); - if (caPrincipal.contains(hostName)) { - isRootCA = true; - } - storeCerts(subCertificate, rootCertificate, isRootCA, conf); - } - - return containsCerts; + if (!cert.isPresent()) { + throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet()); } + return cert.get(); } - public static Map getAllCerts( + private static Map getAllCerts( DBColumnFamilyDefinition columnFamilyDefinition, ColumnFamilyHandle cfHandle, ManagedRocksDB db) throws IOException, RocksDBException { Map allCerts = new HashMap<>(); @@ -165,13 +205,14 @@ private static DBColumnFamilyDefinition getDbColumnFamilyDefinition( return columnFamilyDefinition; } - private static void storeCerts(X509Certificate scmCertificate, - X509Certificate rootCertificate, boolean isRootCA, SecurityConfig securityConfig) + private void storeCerts(X509Certificate scmCertificate, + X509Certificate rootCertificate, boolean isRootCA) throws CertificateException, IOException { + SecurityConfig securityConfig = new SecurityConfig(parent.getOzoneConf()); CertificateCodec certCodec = new CertificateCodec(securityConfig, SCMCertificateClient.COMPONENT_NAME); - LOG.info("Writing certs to path : " + certCodec.getLocation().toString()); + out().println("Writing certs to path : " + certCodec.getLocation().toString()); CertPath certPath = addRootCertInPath(scmCertificate, rootCertificate); CertPath rootCertPath = getRootCertPath(rootCertificate); @@ -191,13 +232,13 @@ private static void storeCerts(X509Certificate scmCertificate, if (isRootCA) { CertificateCodec rootCertCodec = new CertificateCodec(securityConfig, OzoneConsts.SCM_ROOT_CA_COMPONENT_NAME); - LOG.info("Writing root certs to path : " + rootCertCodec.getLocation().toString()); + out().println("Writing root certs to path : " + rootCertCodec.getLocation().toString()); rootCertCodec.writeCertificate(rootCertCodec.getLocation().toAbsolutePath(), securityConfig.getCertificateFileName(), encodedRootCert); } } - public static CertPath addRootCertInPath(X509Certificate scmCert, + public CertPath addRootCertInPath(X509Certificate scmCert, X509Certificate rootCert) throws CertificateException { ArrayList updatedList = new ArrayList<>(); updatedList.add(scmCert); @@ -206,7 +247,7 @@ public static CertPath addRootCertInPath(X509Certificate scmCert, return factory.engineGenerateCertPath(updatedList); } - public static CertPath getRootCertPath(X509Certificate rootCert) + public CertPath getRootCertPath(X509Certificate rootCert) throws CertificateException { ArrayList updatedList = new ArrayList<>(); updatedList.add(rootCert); From 8ba72adc9b742ce0c3449706377e8ae9fd0b3c62 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Fri, 30 Aug 2024 02:30:50 +0530 Subject: [PATCH 5/7] move command under ozone repair and replace certificateFactory usage --- .../ozone/repair/RecoverSCMCertificate.java | 260 ++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/RecoverSCMCertificate.java diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/RecoverSCMCertificate.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/RecoverSCMCertificate.java new file mode 100644 index 00000000000..6f91f6adc86 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/RecoverSCMCertificate.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.repair; + +import org.apache.hadoop.hdds.cli.SubcommandWithParent; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.security.SecurityConfig; +import org.apache.hadoop.hdds.security.x509.certificate.authority.CAType; +import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; +import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; +import org.apache.hadoop.hdds.utils.db.DBDefinition; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; +import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.debug.DBDefinitionFactory; +import org.apache.hadoop.ozone.debug.RocksDBUtils; +import java.security.cert.CertificateFactory; +import org.kohsuke.MetaInfServices; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.RocksDBException; +import picocli.CommandLine; + +import java.io.IOException; +import java.io.PrintWriter; +import java.math.BigInteger; +import java.net.InetAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.security.cert.CertPath; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.Map; +import java.util.HashMap; +import java.util.List; +import java.util.ArrayList; +import java.util.Optional; +import java.util.Arrays; +import java.util.concurrent.Callable; + +import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.VALID_SCM_CERTS; +import static org.apache.hadoop.hdds.security.x509.certificate.client.DefaultCertificateClient.CERT_FILE_NAME_FORMAT; +import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.removeTrailingSlashIfNeeded; + +/** + * In case of accidental deletion of SCM certificates from local storage, + * this tool restores the certs that are persisted into the SCM DB. + * Note that this will only work if the SCM has persisted certs in its RocksDB. + */ +@CommandLine.Command( + name = "cert-recover", + description = "Recover Deleted SCM Certificate from RocksDB") +@MetaInfServices(SubcommandWithParent.class) +public class RecoverSCMCertificate implements Callable, SubcommandWithParent { + + @CommandLine.Option(names = {"--db"}, + required = true, + description = "SCM DB Path") + private String dbPath; + + @CommandLine.ParentCommand + private OzoneRepair parent; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @Override + public Class getParentType() { + return OzoneRepair.class; + } + + private PrintWriter err() { + return spec.commandLine().getErr(); + } + + private PrintWriter out() { + return spec.commandLine().getOut(); + } + + @Override + public Void call() throws Exception { + dbPath = removeTrailingSlashIfNeeded(dbPath); + String tableName = VALID_SCM_CERTS.getName(); + DBDefinition dbDefinition = + DBDefinitionFactory.getDefinition(Paths.get(dbPath), new OzoneConfiguration()); + if (dbDefinition == null) { + throw new Exception("Error: Incorrect DB Path"); + } + DBColumnFamilyDefinition columnFamilyDefinition = + getDbColumnFamilyDefinition(tableName, dbDefinition); + + try { + List cfDescList = RocksDBUtils.getColumnFamilyDescriptors(dbPath); + final List cfHandleList = new ArrayList<>(); + byte[] tableNameBytes = tableName.getBytes(StandardCharsets.UTF_8); + ColumnFamilyHandle cfHandle = null; + try (ManagedRocksDB db = ManagedRocksDB.openReadOnly(dbPath, cfDescList, + cfHandleList)) { + cfHandle = getColumnFamilyHandle(cfHandleList, tableNameBytes); + + Map allCerts = getAllCerts(columnFamilyDefinition, cfHandle, db); + out().println("All Certs in DB : " + allCerts.keySet()); + String hostName = InetAddress.getLocalHost().getCanonicalHostName(); + out().println("Host: " + hostName); + + X509Certificate subCertificate = getSubCertificate(allCerts, hostName); + X509Certificate rootCertificate = getRootCertificate(allCerts); + + out().println("Sub cert serialID for this host: " + subCertificate.getSerialNumber().toString()); + out().println("Root cert serialID: " + rootCertificate.getSerialNumber().toString()); + + boolean isRootCA = false; + + String caPrincipal = rootCertificate.getSubjectDN().getName(); + if (caPrincipal.contains(hostName)) { + isRootCA = true; + } + storeCerts(subCertificate, rootCertificate, isRootCA); + } + } catch (RocksDBException | CertificateException exception) { + err().print("Failed to recover scm cert"); + } + return null; + } + + private static ColumnFamilyHandle getColumnFamilyHandle( + List cfHandleList, byte[] tableNameBytes) throws Exception { + ColumnFamilyHandle cfHandle = null; + for (ColumnFamilyHandle cf : cfHandleList) { + if (Arrays.equals(cf.getName(), tableNameBytes)) { + cfHandle = cf; + break; + } + } + if (cfHandle == null) { + throw new Exception("Error: VALID_SCM_CERTS table not found in DB"); + } + return cfHandle; + } + + private static X509Certificate getRootCertificate( + Map allCerts) throws Exception { + Optional cert = allCerts.values().stream().filter( + c -> c.getSubjectDN().getName() + .contains(OzoneConsts.SCM_ROOT_CA_PREFIX)).findFirst(); + if (!cert.isPresent()) { + throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet()); + } + return cert.get(); + } + + + private static X509Certificate getSubCertificate( + Map allCerts, String hostName) throws Exception { + Optional cert = allCerts.values().stream().filter( + c -> c.getSubjectDN().getName() + .contains(OzoneConsts.SCM_SUB_CA_PREFIX) && c.getSubjectDN() + .getName().contains(hostName)).findFirst(); + if (!cert.isPresent()) { + throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet()); + } + return cert.get(); + } + + private static Map getAllCerts( + DBColumnFamilyDefinition columnFamilyDefinition, + ColumnFamilyHandle cfHandle, ManagedRocksDB db) throws IOException, RocksDBException { + Map allCerts = new HashMap<>(); + ManagedRocksIterator rocksIterator = ManagedRocksIterator.managed(db.get().newIterator(cfHandle)); + rocksIterator.get().seekToFirst(); + while (rocksIterator.get().isValid()) { + BigInteger id = (BigInteger) columnFamilyDefinition.getKeyCodec() + .fromPersistedFormat(rocksIterator.get().key()); + X509Certificate certificate = + (X509Certificate) columnFamilyDefinition.getValueCodec() + .fromPersistedFormat(rocksIterator.get().value()); + allCerts.put(id, certificate); + rocksIterator.get().next(); + } + return allCerts; + } + + private static DBColumnFamilyDefinition getDbColumnFamilyDefinition( + String tableName, DBDefinition dbDefinition) throws Exception { + DBColumnFamilyDefinition columnFamilyDefinition = + dbDefinition.getColumnFamily(tableName); + if (columnFamilyDefinition == null) { + throw new Exception( + "Error: VALID_SCM_CERTS table no found in Definition"); + } + return columnFamilyDefinition; + } + + private void storeCerts(X509Certificate scmCertificate, + X509Certificate rootCertificate, boolean isRootCA) + throws CertificateException, IOException { + SecurityConfig securityConfig = new SecurityConfig(parent.getOzoneConf()); + CertificateCodec certCodec = + new CertificateCodec(securityConfig, SCMCertificateClient.COMPONENT_NAME); + + out().println("Writing certs to path : " + certCodec.getLocation().toString()); + + CertPath certPath = addRootCertInPath(scmCertificate, rootCertificate); + CertPath rootCertPath = getRootCertPath(rootCertificate); + String encodedCert = CertificateCodec.getPEMEncodedString(certPath); + String certName = String.format(CERT_FILE_NAME_FORMAT, + CAType.NONE.getFileNamePrefix() + scmCertificate.getSerialNumber().toString()); + certCodec.writeCertificate(certName, encodedCert); + + String rootCertName = String.format(CERT_FILE_NAME_FORMAT, + CAType.SUBORDINATE.getFileNamePrefix() + rootCertificate.getSerialNumber().toString()); + String encodedRootCert = CertificateCodec.getPEMEncodedString(rootCertPath); + certCodec.writeCertificate(rootCertName, encodedRootCert); + + certCodec.writeCertificate(certCodec.getLocation().toAbsolutePath(), + securityConfig.getCertificateFileName(), encodedCert); + + if (isRootCA) { + CertificateCodec rootCertCodec = + new CertificateCodec(securityConfig, OzoneConsts.SCM_ROOT_CA_COMPONENT_NAME); + out().println("Writing root certs to path : " + rootCertCodec.getLocation().toString()); + rootCertCodec.writeCertificate(rootCertCodec.getLocation().toAbsolutePath(), + securityConfig.getCertificateFileName(), encodedRootCert); + } + } + + public CertPath addRootCertInPath(X509Certificate scmCert, + X509Certificate rootCert) throws CertificateException { + ArrayList updatedList = new ArrayList<>(); + updatedList.add(scmCert); + updatedList.add(rootCert); + CertificateFactory certFactory = + CertificateCodec.getCertFactory(); + return certFactory.generateCertPath(updatedList); + } + + public CertPath getRootCertPath(X509Certificate rootCert) + throws CertificateException { + ArrayList updatedList = new ArrayList<>(); + updatedList.add(rootCert); + CertificateFactory factory = CertificateCodec.getCertFactory(); + return factory.generateCertPath(updatedList); + } +} From a0c258527b3a7ad8ff4ac353079f536d66a4b9c7 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Fri, 30 Aug 2024 02:32:38 +0530 Subject: [PATCH 6/7] remove ozone debug --- .../ozone/debug/RecoverSCMCertificate.java | 257 ------------------ 1 file changed, 257 deletions(-) delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java deleted file mode 100644 index 36b82a21b26..00000000000 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RecoverSCMCertificate.java +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.debug; - -import org.apache.hadoop.hdds.cli.SubcommandWithParent; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.security.SecurityConfig; -import org.apache.hadoop.hdds.security.x509.certificate.authority.CAType; -import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; -import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; -import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; -import org.apache.hadoop.hdds.utils.db.DBDefinition; -import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; -import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; -import org.apache.hadoop.ozone.OzoneConsts; -import org.bouncycastle.jcajce.provider.asymmetric.x509.CertificateFactory; -import org.kohsuke.MetaInfServices; -import org.rocksdb.ColumnFamilyDescriptor; -import org.rocksdb.ColumnFamilyHandle; -import org.rocksdb.RocksDBException; -import picocli.CommandLine; - -import java.io.IOException; -import java.io.PrintWriter; -import java.math.BigInteger; -import java.net.InetAddress; -import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; -import java.security.cert.CertPath; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; -import java.util.Map; -import java.util.HashMap; -import java.util.List; -import java.util.ArrayList; -import java.util.Optional; -import java.util.Arrays; -import java.util.concurrent.Callable; - -import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.VALID_SCM_CERTS; -import static org.apache.hadoop.hdds.security.x509.certificate.client.DefaultCertificateClient.CERT_FILE_NAME_FORMAT; -import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.removeTrailingSlashIfNeeded; - -/** - * In case of accidental deletion of SCM certificates from local storage, - * this tool restores the certs that are persisted into the SCM DB. - * Note that this will only work if the SCM has persisted certs in its RocksDB. - */ -@CommandLine.Command( - name = "cert-recover", - description = "Recover Deleted SCM Certificate from RocksDB") -@MetaInfServices(SubcommandWithParent.class) -public class RecoverSCMCertificate implements Callable, SubcommandWithParent { - - @CommandLine.Option(names = {"--db"}, - required = true, - description = "SCM DB Path") - private String dbPath; - - @CommandLine.ParentCommand - private OzoneDebug parent; - - @CommandLine.Spec - private CommandLine.Model.CommandSpec spec; - - @Override - public Class getParentType() { - return OzoneDebug.class; - } - - private PrintWriter err() { - return spec.commandLine().getErr(); - } - - private PrintWriter out() { - return spec.commandLine().getOut(); - } - - @Override - public Void call() throws Exception { - dbPath = removeTrailingSlashIfNeeded(dbPath); - String tableName = VALID_SCM_CERTS.getName(); - DBDefinition dbDefinition = - DBDefinitionFactory.getDefinition(Paths.get(dbPath), new OzoneConfiguration()); - if (dbDefinition == null) { - throw new Exception("Error: Incorrect DB Path"); - } - DBColumnFamilyDefinition columnFamilyDefinition = - getDbColumnFamilyDefinition(tableName, dbDefinition); - - try { - List cfDescList = RocksDBUtils.getColumnFamilyDescriptors(dbPath); - final List cfHandleList = new ArrayList<>(); - byte[] tableNameBytes = tableName.getBytes(StandardCharsets.UTF_8); - ColumnFamilyHandle cfHandle = null; - try (ManagedRocksDB db = ManagedRocksDB.openReadOnly(dbPath, cfDescList, - cfHandleList)) { - cfHandle = getColumnFamilyHandle(cfHandleList, tableNameBytes); - - Map allCerts = getAllCerts(columnFamilyDefinition, cfHandle, db); - out().println("All Certs in DB : " + allCerts.keySet()); - String hostName = InetAddress.getLocalHost().getCanonicalHostName(); - out().println("Host: " + hostName); - - X509Certificate subCertificate = getSubCertificate(allCerts, hostName); - X509Certificate rootCertificate = getRootCertificate(allCerts); - - out().println("Sub cert serialID for this host: " + subCertificate.getSerialNumber().toString()); - out().println("Root cert serialID: " + rootCertificate.getSerialNumber().toString()); - - boolean isRootCA = false; - - String caPrincipal = rootCertificate.getSubjectDN().getName(); - if (caPrincipal.contains(hostName)) { - isRootCA = true; - } - storeCerts(subCertificate, rootCertificate, isRootCA); - } - } catch (RocksDBException | CertificateException exception) { - err().print("Failed to recover scm cert"); - } - return null; - } - - private static ColumnFamilyHandle getColumnFamilyHandle( - List cfHandleList, byte[] tableNameBytes) throws Exception { - ColumnFamilyHandle cfHandle = null; - for (ColumnFamilyHandle cf : cfHandleList) { - if (Arrays.equals(cf.getName(), tableNameBytes)) { - cfHandle = cf; - break; - } - } - if (cfHandle == null) { - throw new Exception("Error: VALID_SCM_CERTS table not found in DB"); - } - return cfHandle; - } - - private static X509Certificate getRootCertificate( - Map allCerts) throws Exception { - Optional cert = allCerts.values().stream().filter( - c -> c.getSubjectDN().getName() - .contains(OzoneConsts.SCM_ROOT_CA_PREFIX)).findFirst(); - if (!cert.isPresent()) { - throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet()); - } - return cert.get(); - } - - - private static X509Certificate getSubCertificate( - Map allCerts, String hostName) throws Exception { - Optional cert = allCerts.values().stream().filter( - c -> c.getSubjectDN().getName() - .contains(OzoneConsts.SCM_SUB_CA_PREFIX) && c.getSubjectDN() - .getName().contains(hostName)).findFirst(); - if (!cert.isPresent()) { - throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet()); - } - return cert.get(); - } - - private static Map getAllCerts( - DBColumnFamilyDefinition columnFamilyDefinition, - ColumnFamilyHandle cfHandle, ManagedRocksDB db) throws IOException, RocksDBException { - Map allCerts = new HashMap<>(); - ManagedRocksIterator rocksIterator = ManagedRocksIterator.managed(db.get().newIterator(cfHandle)); - rocksIterator.get().seekToFirst(); - while (rocksIterator.get().isValid()) { - BigInteger id = (BigInteger) columnFamilyDefinition.getKeyCodec() - .fromPersistedFormat(rocksIterator.get().key()); - X509Certificate certificate = - (X509Certificate) columnFamilyDefinition.getValueCodec() - .fromPersistedFormat(rocksIterator.get().value()); - allCerts.put(id, certificate); - rocksIterator.get().next(); - } - return allCerts; - } - - private static DBColumnFamilyDefinition getDbColumnFamilyDefinition( - String tableName, DBDefinition dbDefinition) throws Exception { - DBColumnFamilyDefinition columnFamilyDefinition = - dbDefinition.getColumnFamily(tableName); - if (columnFamilyDefinition == null) { - throw new Exception( - "Error: VALID_SCM_CERTS table no found in Definition"); - } - return columnFamilyDefinition; - } - - private void storeCerts(X509Certificate scmCertificate, - X509Certificate rootCertificate, boolean isRootCA) - throws CertificateException, IOException { - SecurityConfig securityConfig = new SecurityConfig(parent.getOzoneConf()); - CertificateCodec certCodec = - new CertificateCodec(securityConfig, SCMCertificateClient.COMPONENT_NAME); - - out().println("Writing certs to path : " + certCodec.getLocation().toString()); - - CertPath certPath = addRootCertInPath(scmCertificate, rootCertificate); - CertPath rootCertPath = getRootCertPath(rootCertificate); - String encodedCert = CertificateCodec.getPEMEncodedString(certPath); - String certName = String.format(CERT_FILE_NAME_FORMAT, - CAType.NONE.getFileNamePrefix() + scmCertificate.getSerialNumber().toString()); - certCodec.writeCertificate(certName, encodedCert); - - String rootCertName = String.format(CERT_FILE_NAME_FORMAT, - CAType.SUBORDINATE.getFileNamePrefix() + rootCertificate.getSerialNumber().toString()); - String encodedRootCert = CertificateCodec.getPEMEncodedString(rootCertPath); - certCodec.writeCertificate(rootCertName, encodedRootCert); - - certCodec.writeCertificate(certCodec.getLocation().toAbsolutePath(), - securityConfig.getCertificateFileName(), encodedCert); - - if (isRootCA) { - CertificateCodec rootCertCodec = - new CertificateCodec(securityConfig, OzoneConsts.SCM_ROOT_CA_COMPONENT_NAME); - out().println("Writing root certs to path : " + rootCertCodec.getLocation().toString()); - rootCertCodec.writeCertificate(rootCertCodec.getLocation().toAbsolutePath(), - securityConfig.getCertificateFileName(), encodedRootCert); - } - } - - public CertPath addRootCertInPath(X509Certificate scmCert, - X509Certificate rootCert) throws CertificateException { - ArrayList updatedList = new ArrayList<>(); - updatedList.add(scmCert); - updatedList.add(rootCert); - CertificateFactory factory = new CertificateFactory(); - return factory.engineGenerateCertPath(updatedList); - } - - public CertPath getRootCertPath(X509Certificate rootCert) - throws CertificateException { - ArrayList updatedList = new ArrayList<>(); - updatedList.add(rootCert); - CertificateFactory factory = new CertificateFactory(); - return factory.engineGenerateCertPath(updatedList); - } -} From 2f3a9ec97c3e5e2758a75251944c95198d903ab0 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Fri, 30 Aug 2024 14:56:04 +0530 Subject: [PATCH 7/7] fix test failure --- .../src/main/compose/ozonesecure-ha/docker-config | 1 + .../hadoop/ozone/repair/RecoverSCMCertificate.java | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config index db517a7f7c6..38cc5b71a18 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config @@ -33,6 +33,7 @@ OZONE-SITE.XML_ozone.om.http-address.omservice.om3=om3 OZONE-SITE.XML_ozone.om.ratis.enable=true OZONE-SITE.XML_ozone.scm.service.ids=scmservice +OZONE-SITE.XML_ozone.scm.primordial.node.id=scm1 OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3 OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1.org OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2.org diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/RecoverSCMCertificate.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/RecoverSCMCertificate.java index 6f91f6adc86..aca41844a18 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/RecoverSCMCertificate.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/RecoverSCMCertificate.java @@ -61,7 +61,8 @@ /** * In case of accidental deletion of SCM certificates from local storage, * this tool restores the certs that are persisted into the SCM DB. - * Note that this will only work if the SCM has persisted certs in its RocksDB. + * Note that this will only work if the SCM has persisted certs in its RocksDB + * and private keys of the SCM are intact. */ @CommandLine.Command( name = "cert-recover", @@ -113,10 +114,11 @@ public Void call() throws Exception { try (ManagedRocksDB db = ManagedRocksDB.openReadOnly(dbPath, cfDescList, cfHandleList)) { cfHandle = getColumnFamilyHandle(cfHandleList, tableNameBytes); + SecurityConfig securityConfig = new SecurityConfig(parent.getOzoneConf()); Map allCerts = getAllCerts(columnFamilyDefinition, cfHandle, db); out().println("All Certs in DB : " + allCerts.keySet()); - String hostName = InetAddress.getLocalHost().getCanonicalHostName(); + String hostName = InetAddress.getLocalHost().getHostName(); out().println("Host: " + hostName); X509Certificate subCertificate = getSubCertificate(allCerts, hostName); @@ -131,7 +133,7 @@ public Void call() throws Exception { if (caPrincipal.contains(hostName)) { isRootCA = true; } - storeCerts(subCertificate, rootCertificate, isRootCA); + storeCerts(subCertificate, rootCertificate, isRootCA, securityConfig); } } catch (RocksDBException | CertificateException exception) { err().print("Failed to recover scm cert"); @@ -173,7 +175,7 @@ private static X509Certificate getSubCertificate( .contains(OzoneConsts.SCM_SUB_CA_PREFIX) && c.getSubjectDN() .getName().contains(hostName)).findFirst(); if (!cert.isPresent()) { - throw new Exception("Root CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet()); + throw new Exception("Sub CA Cert not found in the DB for this host, Certs in the DB : " + allCerts.keySet()); } return cert.get(); } @@ -208,9 +210,8 @@ private static DBColumnFamilyDefinition getDbColumnFamilyDefinition( } private void storeCerts(X509Certificate scmCertificate, - X509Certificate rootCertificate, boolean isRootCA) + X509Certificate rootCertificate, boolean isRootCA, SecurityConfig securityConfig) throws CertificateException, IOException { - SecurityConfig securityConfig = new SecurityConfig(parent.getOzoneConf()); CertificateCodec certCodec = new CertificateCodec(securityConfig, SCMCertificateClient.COMPONENT_NAME);