HDDS-11371. Handle cases where OM does not have getServerDefaults() i…

…mplemented. (apache#7130) Co-authored-by: saketa <schalamchala@cloudera.com>
aswinshakil · Sep 27, 2024 · 56ddb85 · 56ddb85
1 parent b5097c7
commit 56ddb85
Show file tree

Hide file tree

Showing 15 changed files with 307 additions and 71 deletions.
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -2590,17 +2590,27 @@ public OzoneFsServerDefaults getServerDefaults() throws IOException {
     long now = Time.monotonicNow();
     if ((serverDefaults == null) ||
         (now - serverDefaultsLastUpdate > serverDefaultsValidityPeriod)) {
-      serverDefaults = ozoneManagerClient.getServerDefaults();
-      serverDefaultsLastUpdate = now;
+      try {
+        for (ServiceInfo si : ozoneManagerClient.getServiceInfo()
+            .getServiceInfoList()) {
+          if (si.getServerDefaults() != null) {
+            serverDefaults = si.getServerDefaults();
+            serverDefaultsLastUpdate = now;
+            break;
+          }
+        }
+      } catch (Exception e) {
+        LOG.warn("Could not get server defaults from OM.", e);
+      }
     }
-    assert serverDefaults != null;
     return serverDefaults;
   }
 
   @Override
   public URI getKeyProviderUri() throws IOException {
-    return OzoneKMSUtil.getKeyProviderUri(ugi,
-        null, getServerDefaults().getKeyProviderUri(), conf);
+    String keyProviderUri = (getServerDefaults() != null) ?
+        serverDefaults.getKeyProviderUri() : null;
+    return OzoneKMSUtil.getKeyProviderUri(ugi, null, keyProviderUri, conf);
   }
 
   @Override

diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -278,7 +278,6 @@ public static boolean isReadOnly(
     case SetSafeMode:
     case PrintCompactionLogDag:
     case GetSnapshotInfo:
-    case GetServerDefaults:
     case GetQuotaRepairStatus:
     case StartQuotaRepair:
       return true;

diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java
@@ -25,6 +25,7 @@
 import java.util.Map;
 
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
+import org.apache.hadoop.ozone.OzoneFsServerDefaults;
 import org.apache.hadoop.ozone.OzoneManagerVersion;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
@@ -59,6 +60,7 @@ public final class ServiceInfo {
   private Map<ServicePort.Type, Integer> ports;
 
   private OMRoleInfo omRoleInfo;
+  private OzoneFsServerDefaults serverDefaults;
 
   /**
    * Default constructor for JSON deserialization.
@@ -76,6 +78,24 @@ private ServiceInfo(NodeType nodeType,
                       List<ServicePort> portList,
                       OzoneManagerVersion omVersion,
                       OMRoleInfo omRole) {
+    this(nodeType, hostname, portList, omVersion, omRole, null);
+  }
+
+  /**
+   * Constructs the ServiceInfo for the {@code nodeType}.
+   * @param nodeType type of node/service
+   * @param hostname hostname of the service
+   * @param portList list of ports the service listens to
+   * @param omVersion Om Version
+   * @param omRole OM role Ino
+   * @param keyProviderUri KMS provider URI
+   */
+  private ServiceInfo(NodeType nodeType,
+                      String hostname,
+                      List<ServicePort> portList,
+                      OzoneManagerVersion omVersion,
+                      OMRoleInfo omRole,
+                      OzoneFsServerDefaults serverDefaults) {
     Preconditions.checkNotNull(nodeType);
     Preconditions.checkNotNull(hostname);
     this.nodeType = nodeType;
@@ -86,6 +106,7 @@ private ServiceInfo(NodeType nodeType,
       ports.put(port.getType(), port.getValue());
     }
     this.omRoleInfo = omRole;
+    this.serverDefaults = serverDefaults;
   }
 
   /**
@@ -143,6 +164,15 @@ public OMRoleInfo getOmRoleInfo() {
     return omRoleInfo;
   }
 
+  /**
+   * Returns the Ozone Server default configuration.
+   * @return OmRoleInfo
+   */
+  @JsonIgnore
+  public OzoneFsServerDefaults getServerDefaults() {
+    return serverDefaults;
+  }
+
   /**
    * Converts {@link ServiceInfo} to OzoneManagerProtocolProtos.ServiceInfo.
    *
@@ -170,6 +200,9 @@ public OzoneManagerProtocolProtos.ServiceInfo getProtobuf() {
     if (nodeType == NodeType.OM && omRoleInfo != null) {
       builder.setOmRole(omRoleInfo);
     }
+    if (serverDefaults != null) {
+      builder.setServerDefaults(serverDefaults.getProtobuf());
+    }
     return builder.build();
   }
 
@@ -185,7 +218,9 @@ public static ServiceInfo getFromProtobuf(
         serviceInfo.getHostname(),
         serviceInfo.getServicePortsList(),
         OzoneManagerVersion.fromProtoValue(serviceInfo.getOMVersion()),
-        serviceInfo.hasOmRole() ? serviceInfo.getOmRole() : null);
+        serviceInfo.hasOmRole() ? serviceInfo.getOmRole() : null,
+        serviceInfo.hasServerDefaults() ? OzoneFsServerDefaults.getFromProtobuf(
+            serviceInfo.getServerDefaults()) : null);
   }
 
   /**
@@ -206,6 +241,7 @@ public static class Builder {
     private List<ServicePort> portList = new ArrayList<>();
     private OMRoleInfo omRoleInfo;
     private OzoneManagerVersion omVersion;
+    private OzoneFsServerDefaults serverDefaults;
 
     /**
      * Gets the Om Client Protocol Version.
@@ -259,6 +295,11 @@ public Builder setOmRoleInfo(OMRoleInfo omRole) {
       return this;
     }
 
+    public Builder setServerDefaults(OzoneFsServerDefaults defaults) {
+      serverDefaults = defaults;
+      return this;
+    }
+
     /**
      * Builds and returns {@link ServiceInfo} with the set values.
      * @return {@link ServiceInfo}
@@ -268,7 +309,8 @@ public ServiceInfo build() {
           host,
           portList,
           omVersion,
-          omRoleInfo);
+          omRoleInfo,
+          serverDefaults);
     }
   }
 

diff --git a/...-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/...-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.fs.SafeModeAction;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
 import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.OzoneFsServerDefaults;
 import org.apache.hadoop.ozone.om.IOmMetadataReader;
 import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -1146,14 +1145,6 @@ void setTimes(OmKeyArgs keyArgs, long mtime, long atime)
   boolean setSafeMode(SafeModeAction action, boolean isChecked)
       throws IOException;
 
-  /**
-   * Get server default configurations.
-   *
-   * @return OzoneFsServerDefaults some default configurations from server.
-   * @throws IOException
-   */
-  OzoneFsServerDefaults getServerDefaults() throws IOException;
-
   /**
    * Get status of last triggered quota repair in OM.
    * @return String

diff --git a/...ava/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/...ava/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -41,7 +41,6 @@
 import org.apache.hadoop.ipc.CallerContext;
 import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.OzoneFsServerDefaults;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.ErrorInfo;
@@ -193,8 +192,6 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Secret;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SafeMode;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServerDefaultsRequest;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServerDefaultsResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetAclRequest;
@@ -2559,22 +2556,6 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked)
     return setSafeModeResponse.getResponse();
   }
 
-  @Override
-  public OzoneFsServerDefaults getServerDefaults()
-      throws IOException {
-    ServerDefaultsRequest serverDefaultsRequest =
-        ServerDefaultsRequest.newBuilder().build();
-
-    OMRequest omRequest = createOMRequest(Type.GetServerDefaults)
-        .setServerDefaultsRequest(serverDefaultsRequest).build();
-
-    ServerDefaultsResponse serverDefaultsResponse =
-        handleError(submitRequest(omRequest)).getServerDefaultsResponse();
-
-    return OzoneFsServerDefaults.getFromProtobuf(
-        serverDefaultsResponse.getServerDefaults());
-  }
-
   @Override
   public String getQuotaRepairStatus() throws IOException {
     OzoneManagerProtocolProtos.GetQuotaRepairStatusRequest quotaRepairStatusRequest =

diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/.env b/hadoop-ozone/dist/src/main/compose/xcompat/.env
@@ -17,3 +17,5 @@
 HDDS_VERSION=${hdds.version}
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner
+HADOOP_VERSION=${hadoop.version}
+OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml
@@ -21,41 +21,53 @@ services:
       - docker-config
     volumes:
       - ../..:/opt/ozone
+      - ../_keytabs:/etc/security/keytabs
+      - ./krb5.conf:/etc/krb5.conf
     command: ["sleep","1000000"]
   old_client_1_1_0:
     image: apache/ozone:1.1.0
     env_file:
       - docker-config
     volumes:
       - ../..:/opt/ozone
+      - ../_keytabs:/etc/security/keytabs
+      - ./krb5.conf:/etc/krb5.conf
     command: ["sleep","1000000"]
   old_client_1_2_1:
     image: apache/ozone:1.2.1
     env_file:
       - docker-config
     volumes:
       - ../..:/opt/ozone
+      - ../_keytabs:/etc/security/keytabs
+      - ./krb5.conf:/etc/krb5.conf
     command: ["sleep","1000000"]
   old_client_1_3_0:
     image: apache/ozone:1.3.0
     env_file:
       - docker-config
     volumes:
       - ../..:/opt/ozone
+      - ../_keytabs:/etc/security/keytabs
+      - ./krb5.conf:/etc/krb5.conf
     command: ["sleep","1000000"]
   old_client_1_4_0:
     image: apache/ozone:1.4.0
     env_file:
       - docker-config
     volumes:
       - ../..:/opt/ozone
+      - ../_keytabs:/etc/security/keytabs
+      - ./krb5.conf:/etc/krb5.conf
     command: ["sleep","1000000"]
   new_client:
     image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
     env_file:
       - docker-config
     volumes:
       - ../..:/opt/hadoop
+      - ../_keytabs:/etc/security/keytabs
+      - ./krb5.conf:/etc/krb5.conf
     environment:
       OZONE_OPTS:
     command: ["sleep","1000000"]
diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/docker-config b/hadoop-ozone/dist/src/main/compose/xcompat/docker-config
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+CORE-SITE.XML_fs.defaultFS=ofs://om
+CORE-SITE.XML_fs.trash.interval=1
 CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem
 
 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
@@ -22,6 +24,7 @@ OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.http-address=scm:9876
 OZONE-SITE.XML_ozone.recon.address=recon:9891
 OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
 OZONE-SITE.XML_ozone.server.default.replication=3
@@ -31,9 +34,98 @@ OZONE-SITE.XML_ozone.scm.container.size=1GB
 OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
 OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
 OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
 OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
+OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
 OZONE-SITE.XML_recon.om.snapshot.task.interval.delay=1m
 OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
 OZONE-SITE.XML_ozone.default.bucket.layout=LEGACY
 OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
-no_proxy=om,recon,scm,s3g,kdc,localhost,127.0.0.1
+
+OZONE-SITE.XML_hdds.block.token.enabled=true
+OZONE-SITE.XML_hdds.container.token.enabled=true
+OZONE-SITE.XML_hdds.grpc.tls.enabled=true
+
+OZONE-SITE.XML_ozone.security.enabled=true
+OZONE-SITE.XML_ozone.acl.enabled=true
+OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer
+OZONE-SITE.XML_ozone.administrators="testuser,recon,om"
+OZONE-SITE.XML_ozone.s3.administrators="testuser,recon,om"
+OZONE-SITE.XML_ozone.recon.administrators="testuser2"
+OZONE-SITE.XML_ozone.s3.administrators="testuser,s3g"
+
+HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
+HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
+CORE-SITE.XML_dfs.data.transfer.protection=authentication
+CORE-SITE.XML_hadoop.security.authentication=kerberos
+CORE-SITE.XML_hadoop.security.auth_to_local="DEFAULT"
+CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms
+
+OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
+OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab
+OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM
+
+OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab
+OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM
+
+OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab
+OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/httpfs@EXAMPLE.COM
+
+HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM
+HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+
+OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
+OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true
+OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer
+
+OZONE-SITE.XML_ozone.om.http.auth.type=kerberos
+OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos
+OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.s3g.secret.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.httpfs.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos
+
+OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/om.keytab
+OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/dn@EXAMPLE.COM
+OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/dn.keytab
+OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/s3g.keytab
+OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM
+OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
+OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal=HTTP/httpfs@EXAMPLE.COM
+OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=*
+OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab
+
+CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed=false
+CORE-SITE.XML_hadoop.http.authentication.signature.secret.file=/etc/security/http_secret
+CORE-SITE.XML_hadoop.http.authentication.type=kerberos
+CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM
+CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+
+CORE-SITE.XML_hadoop.security.authorization=true
+HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=*
+HADOOP-POLICY.XML_ozone.security.reconfigure.protocol.acl=*
+
+KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=*
+KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=*
+KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=*
+
+OZONE_DATANODE_SECURE_USER=root
+JSVC_HOME=/usr/bin
+
+OZONE_LOG_DIR=/var/log/hadoop
+
+no_proxy=om,scm,recon,s3g,kdc,localhost,127.0.0.1