From 56ddb85d94a1b0693cf6feacfd26e682f9073380 Mon Sep 17 00:00:00 2001 From: SaketaChalamchala Date: Fri, 27 Sep 2024 14:44:33 -0700 Subject: [PATCH] HDDS-11371. Handle cases where OM does not have getServerDefaults() implemented. (#7130) Co-authored-by: saketa --- .../hadoop/ozone/client/rpc/RpcClient.java | 20 +++- .../java/org/apache/hadoop/ozone/OmUtils.java | 1 - .../hadoop/ozone/om/helpers/ServiceInfo.java | 46 ++++++++- .../om/protocol/OzoneManagerProtocol.java | 9 -- ...ManagerProtocolClientSideTranslatorPB.java | 19 ---- .../dist/src/main/compose/xcompat/.env | 2 + .../src/main/compose/xcompat/clients.yaml | 12 +++ .../src/main/compose/xcompat/docker-config | 94 ++++++++++++++++++- .../dist/src/main/compose/xcompat/krb5.conf | 41 ++++++++ .../src/main/compose/xcompat/new-cluster.yaml | 33 ++++++- .../src/main/compose/xcompat/old-cluster.yaml | 31 ++++++ .../dist/src/main/compose/xcompat/test.sh | 29 +++++- .../src/main/proto/OmClientProtocol.proto | 11 +-- .../apache/hadoop/ozone/om/OzoneManager.java | 23 ++--- .../OzoneManagerRequestHandler.java | 7 -- 15 files changed, 307 insertions(+), 71 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/compose/xcompat/krb5.conf diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 9dc11637f3c..fe986640176 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -2590,17 +2590,27 @@ public OzoneFsServerDefaults getServerDefaults() throws IOException { long now = Time.monotonicNow(); if ((serverDefaults == null) || (now - serverDefaultsLastUpdate > serverDefaultsValidityPeriod)) { - serverDefaults = ozoneManagerClient.getServerDefaults(); - serverDefaultsLastUpdate = now; + try { + for (ServiceInfo si : ozoneManagerClient.getServiceInfo() + .getServiceInfoList()) { + if (si.getServerDefaults() != null) { + serverDefaults = si.getServerDefaults(); + serverDefaultsLastUpdate = now; + break; + } + } + } catch (Exception e) { + LOG.warn("Could not get server defaults from OM.", e); + } } - assert serverDefaults != null; return serverDefaults; } @Override public URI getKeyProviderUri() throws IOException { - return OzoneKMSUtil.getKeyProviderUri(ugi, - null, getServerDefaults().getKeyProviderUri(), conf); + String keyProviderUri = (getServerDefaults() != null) ? + serverDefaults.getKeyProviderUri() : null; + return OzoneKMSUtil.getKeyProviderUri(ugi, null, keyProviderUri, conf); } @Override diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index b70ea51fde5..2235e11de2b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -278,7 +278,6 @@ public static boolean isReadOnly( case SetSafeMode: case PrintCompactionLogDag: case GetSnapshotInfo: - case GetServerDefaults: case GetQuotaRepairStatus: case StartQuotaRepair: return true; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java index c8bdbf43c42..5dbe3487e19 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; +import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo; @@ -59,6 +60,7 @@ public final class ServiceInfo { private Map ports; private OMRoleInfo omRoleInfo; + private OzoneFsServerDefaults serverDefaults; /** * Default constructor for JSON deserialization. @@ -76,6 +78,24 @@ private ServiceInfo(NodeType nodeType, List portList, OzoneManagerVersion omVersion, OMRoleInfo omRole) { + this(nodeType, hostname, portList, omVersion, omRole, null); + } + + /** + * Constructs the ServiceInfo for the {@code nodeType}. + * @param nodeType type of node/service + * @param hostname hostname of the service + * @param portList list of ports the service listens to + * @param omVersion Om Version + * @param omRole OM role Ino + * @param keyProviderUri KMS provider URI + */ + private ServiceInfo(NodeType nodeType, + String hostname, + List portList, + OzoneManagerVersion omVersion, + OMRoleInfo omRole, + OzoneFsServerDefaults serverDefaults) { Preconditions.checkNotNull(nodeType); Preconditions.checkNotNull(hostname); this.nodeType = nodeType; @@ -86,6 +106,7 @@ private ServiceInfo(NodeType nodeType, ports.put(port.getType(), port.getValue()); } this.omRoleInfo = omRole; + this.serverDefaults = serverDefaults; } /** @@ -143,6 +164,15 @@ public OMRoleInfo getOmRoleInfo() { return omRoleInfo; } + /** + * Returns the Ozone Server default configuration. + * @return OmRoleInfo + */ + @JsonIgnore + public OzoneFsServerDefaults getServerDefaults() { + return serverDefaults; + } + /** * Converts {@link ServiceInfo} to OzoneManagerProtocolProtos.ServiceInfo. * @@ -170,6 +200,9 @@ public OzoneManagerProtocolProtos.ServiceInfo getProtobuf() { if (nodeType == NodeType.OM && omRoleInfo != null) { builder.setOmRole(omRoleInfo); } + if (serverDefaults != null) { + builder.setServerDefaults(serverDefaults.getProtobuf()); + } return builder.build(); } @@ -185,7 +218,9 @@ public static ServiceInfo getFromProtobuf( serviceInfo.getHostname(), serviceInfo.getServicePortsList(), OzoneManagerVersion.fromProtoValue(serviceInfo.getOMVersion()), - serviceInfo.hasOmRole() ? serviceInfo.getOmRole() : null); + serviceInfo.hasOmRole() ? serviceInfo.getOmRole() : null, + serviceInfo.hasServerDefaults() ? OzoneFsServerDefaults.getFromProtobuf( + serviceInfo.getServerDefaults()) : null); } /** @@ -206,6 +241,7 @@ public static class Builder { private List portList = new ArrayList<>(); private OMRoleInfo omRoleInfo; private OzoneManagerVersion omVersion; + private OzoneFsServerDefaults serverDefaults; /** * Gets the Om Client Protocol Version. @@ -259,6 +295,11 @@ public Builder setOmRoleInfo(OMRoleInfo omRole) { return this; } + public Builder setServerDefaults(OzoneFsServerDefaults defaults) { + serverDefaults = defaults; + return this; + } + /** * Builds and returns {@link ServiceInfo} with the set values. * @return {@link ServiceInfo} @@ -268,7 +309,8 @@ public ServiceInfo build() { host, portList, omVersion, - omRoleInfo); + omRoleInfo, + serverDefaults); } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 37481b00ea2..79ddbbf8dad 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.SafeModeAction; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.om.IOmMetadataReader; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -1146,14 +1145,6 @@ void setTimes(OmKeyArgs keyArgs, long mtime, long atime) boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException; - /** - * Get server default configurations. - * - * @return OzoneFsServerDefaults some default configurations from server. - * @throws IOException - */ - OzoneFsServerDefaults getServerDefaults() throws IOException; - /** * Get status of last triggered quota repair in OM. * @return String diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 432b55051da..b140cf95e69 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -41,7 +41,6 @@ import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.OzoneFsServerDefaults; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.ErrorInfo; @@ -193,8 +192,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Secret; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SafeMode; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServerDefaultsRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServerDefaultsResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetAclRequest; @@ -2559,22 +2556,6 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) return setSafeModeResponse.getResponse(); } - @Override - public OzoneFsServerDefaults getServerDefaults() - throws IOException { - ServerDefaultsRequest serverDefaultsRequest = - ServerDefaultsRequest.newBuilder().build(); - - OMRequest omRequest = createOMRequest(Type.GetServerDefaults) - .setServerDefaultsRequest(serverDefaultsRequest).build(); - - ServerDefaultsResponse serverDefaultsResponse = - handleError(submitRequest(omRequest)).getServerDefaultsResponse(); - - return OzoneFsServerDefaults.getFromProtobuf( - serverDefaultsResponse.getServerDefaults()); - } - @Override public String getQuotaRepairStatus() throws IOException { OzoneManagerProtocolProtos.GetQuotaRepairStatusRequest quotaRepairStatusRequest = diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/.env b/hadoop-ozone/dist/src/main/compose/xcompat/.env index 140975d4bd0..a673b7f4655 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/.env +++ b/hadoop-ozone/dist/src/main/compose/xcompat/.env @@ -17,3 +17,5 @@ HDDS_VERSION=${hdds.version} OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner +HADOOP_VERSION=${hadoop.version} +OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image} diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml index 0bf0f619bd7..bfb346f0747 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml +++ b/hadoop-ozone/dist/src/main/compose/xcompat/clients.yaml @@ -21,6 +21,8 @@ services: - docker-config volumes: - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf command: ["sleep","1000000"] old_client_1_1_0: image: apache/ozone:1.1.0 @@ -28,6 +30,8 @@ services: - docker-config volumes: - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf command: ["sleep","1000000"] old_client_1_2_1: image: apache/ozone:1.2.1 @@ -35,6 +39,8 @@ services: - docker-config volumes: - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf command: ["sleep","1000000"] old_client_1_3_0: image: apache/ozone:1.3.0 @@ -42,6 +48,8 @@ services: - docker-config volumes: - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf command: ["sleep","1000000"] old_client_1_4_0: image: apache/ozone:1.4.0 @@ -49,6 +57,8 @@ services: - docker-config volumes: - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf command: ["sleep","1000000"] new_client: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} @@ -56,6 +66,8 @@ services: - docker-config volumes: - ../..:/opt/hadoop + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf environment: OZONE_OPTS: command: ["sleep","1000000"] diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/docker-config b/hadoop-ozone/dist/src/main/compose/xcompat/docker-config index 85099f902d3..1a61aaf4f7e 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/docker-config +++ b/hadoop-ozone/dist/src/main/compose/xcompat/docker-config @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +CORE-SITE.XML_fs.defaultFS=ofs://om +CORE-SITE.XML_fs.trash.interval=1 CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem OZONE-SITE.XML_hdds.datanode.dir=/data/hdds @@ -22,6 +24,7 @@ OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 +OZONE-SITE.XML_ozone.scm.http-address=scm:9876 OZONE-SITE.XML_ozone.recon.address=recon:9891 OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon OZONE-SITE.XML_ozone.server.default.replication=3 @@ -31,9 +34,98 @@ OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 +OZONE-SITE.XML_ozone.datanode.pipeline.limit=1 OZONE-SITE.XML_recon.om.snapshot.task.interval.delay=1m OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s OZONE-SITE.XML_ozone.default.bucket.layout=LEGACY OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http -no_proxy=om,recon,scm,s3g,kdc,localhost,127.0.0.1 + +OZONE-SITE.XML_hdds.block.token.enabled=true +OZONE-SITE.XML_hdds.container.token.enabled=true +OZONE-SITE.XML_hdds.grpc.tls.enabled=true + +OZONE-SITE.XML_ozone.security.enabled=true +OZONE-SITE.XML_ozone.acl.enabled=true +OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer +OZONE-SITE.XML_ozone.administrators="testuser,recon,om" +OZONE-SITE.XML_ozone.s3.administrators="testuser,recon,om" +OZONE-SITE.XML_ozone.recon.administrators="testuser2" +OZONE-SITE.XML_ozone.s3.administrators="testuser,s3g" + +HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019 +HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012 +CORE-SITE.XML_dfs.data.transfer.protection=authentication +CORE-SITE.XML_hadoop.security.authentication=kerberos +CORE-SITE.XML_hadoop.security.auth_to_local="DEFAULT" +CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms + +OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM +OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab +OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM +OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab +OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab +OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM + +OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab +OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM + +OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab +OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/httpfs@EXAMPLE.COM + +HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM +HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab +HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab +HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM +HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab + +OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true +OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true +OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer + +OZONE-SITE.XML_ozone.om.http.auth.type=kerberos +OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos +OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos +OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos +OZONE-SITE.XML_ozone.s3g.secret.http.auth.type=kerberos +OZONE-SITE.XML_ozone.httpfs.http.auth.type=kerberos +OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos + +OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM +OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/scm.keytab +OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/om@EXAMPLE.COM +OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/om.keytab +OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/dn@EXAMPLE.COM +OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/dn.keytab +OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/s3g.keytab +OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM +OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab=/etc/security/keytabs/httpfs.keytab +OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal=HTTP/httpfs@EXAMPLE.COM +OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=* +OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab + +CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed=false +CORE-SITE.XML_hadoop.http.authentication.signature.secret.file=/etc/security/http_secret +CORE-SITE.XML_hadoop.http.authentication.type=kerberos +CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM +CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab + +CORE-SITE.XML_hadoop.security.authorization=true +HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=* +HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=* +HADOOP-POLICY.XML_ozone.security.reconfigure.protocol.acl=* + +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=* +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=* +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=* + +OZONE_DATANODE_SECURE_USER=root +JSVC_HOME=/usr/bin + +OZONE_LOG_DIR=/var/log/hadoop + +no_proxy=om,scm,recon,s3g,kdc,localhost,127.0.0.1 diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/krb5.conf b/hadoop-ozone/dist/src/main/compose/xcompat/krb5.conf new file mode 100644 index 00000000000..eefc5b9c685 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/xcompat/krb5.conf @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[logging] +default = FILE:/var/log/krb5libs.log +kdc = FILE:/var/log/krb5kdc.log +admin_server = FILE:/var/log/kadmind.log + +[libdefaults] + dns_canonicalize_hostname = false + dns_lookup_realm = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + rdns = false + default_realm = EXAMPLE.COM + +[realms] + EXAMPLE.COM = { + kdc = kdc + admin_server = kdc + max_renewable_life = 7d + } + +[domain_realm] + .example.com = EXAMPLE.COM + example.com = EXAMPLE.COM + diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml index 6e3ff6cfbc9..32059140ce9 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml +++ b/hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml @@ -18,14 +18,39 @@ x-new-config: &new-config image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} + dns_search: . env_file: - docker-config volumes: - ../..:/opt/hadoop + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf services: + kdc: + image: ${OZONE_TESTKRB5_IMAGE} + hostname: kdc + dns_search: . + volumes: + - ../..:/opt/hadoop + - ../_keytabs:/etc/security/keytabs + command: [ "krb5kdc","-n" ] + kms: + image: apache/hadoop:${HADOOP_VERSION} + hostname: kms + dns_search: . + ports: + - 9600:9600 + env_file: + - ./docker-config + environment: + HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop + volumes: + - ../../libexec/transformation.py:/opt/transformation.py + command: [ "hadoop", "kms" ] datanode: <<: *new-config + hostname: dn ports: - 19864 - 9882 @@ -34,15 +59,17 @@ services: command: ["ozone","datanode"] om: <<: *new-config + hostname: om environment: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - OZONE_OPTS: + OZONE_OPTS: -Dcom.sun.net.ssl.checkRevocation=false ports: - 9874:9874 - 9862:9862 command: ["ozone","om"] recon: <<: *new-config + hostname: recon ports: - 9888:9888 environment: @@ -50,6 +77,7 @@ services: command: ["ozone","recon"] s3g: <<: *new-config + hostname: s3g environment: OZONE_OPTS: ports: @@ -57,9 +85,12 @@ services: command: ["ozone","s3g"] scm: <<: *new-config + hostname: scm ports: - 9876:9876 + - 9860:9860 environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" OZONE_OPTS: command: ["ozone","scm"] diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml b/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml index c603bb51df3..941cbc14df1 100644 --- a/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml +++ b/hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml @@ -18,14 +18,39 @@ x-old-config: &old-config image: apache/ozone:${OZONE_VERSION} + dns_search: . env_file: - docker-config volumes: - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + - ./krb5.conf:/etc/krb5.conf services: + kdc: + image: ${OZONE_TESTKRB5_IMAGE} + hostname: kdc + dns_search: . + volumes: + - ../..:/opt/ozone + - ../_keytabs:/etc/security/keytabs + command: [ "krb5kdc","-n" ] + kms: + image: apache/hadoop:${HADOOP_VERSION} + hostname: kms + dns_search: . + ports: + - 9600:9600 + env_file: + - ./docker-config + environment: + HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop + volumes: + - ../../libexec/transformation.py:/opt/transformation.py + command: [ "hadoop", "kms" ] datanode: <<: *old-config + hostname: dn ports: - 19864 - 9882 @@ -34,8 +59,10 @@ services: command: ["ozone","datanode"] om: <<: *old-config + hostname: om environment: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + OZONE_OPTS: -Dcom.sun.net.ssl.checkRevocation=false HADOOP_OPTS: ports: - 9874:9874 @@ -43,6 +70,7 @@ services: command: ["ozone","om"] recon: <<: *old-config + hostname: recon ports: - 9888:9888 environment: @@ -50,6 +78,7 @@ services: command: ["ozone","recon"] s3g: <<: *old-config + hostname: s3g environment: HADOOP_OPTS: ports: @@ -57,9 +86,11 @@ services: command: ["ozone","s3g"] scm: <<: *old-config + hostname: scm ports: - 9876:9876 environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-1}" HADOOP_OPTS: command: ["ozone","scm"] diff --git a/hadoop-ozone/dist/src/main/compose/xcompat/test.sh b/hadoop-ozone/dist/src/main/compose/xcompat/test.sh index 695d8bf06ab..8774cf2f632 100755 --- a/hadoop-ozone/dist/src/main/compose/xcompat/test.sh +++ b/hadoop-ozone/dist/src/main/compose/xcompat/test.sh @@ -22,11 +22,15 @@ export COMPOSE_DIR basename=$(basename ${COMPOSE_DIR}) current_version="${ozone.version}" -old_versions="1.0.0 1.1.0 1.2.1 1.3.0 1.4.0" # container is needed for each version in clients.yaml +# TODO: debug acceptance test failures for client versions 1.0.0 on secure clusters +old_versions="1.1.0 1.2.1 1.3.0 1.4.0" # container is needed for each version in clients.yaml # shellcheck source=hadoop-ozone/dist/src/main/compose/testlib.sh source "${COMPOSE_DIR}/../testlib.sh" +export SECURITY_ENABLED=true +: ${OZONE_BUCKET_KEY_NAME:=key1} + old_client() { OZONE_DIR=/opt/ozone container=${client} @@ -40,24 +44,40 @@ new_client() { "$@" } +_kinit() { + execute_command_in_container ${container} kinit -k -t /etc/security/keytabs/testuser.keytab testuser/scm@EXAMPLE.COM +} + _init() { + _kinit execute_command_in_container ${container} ozone freon ockg -n1 -t1 -p warmup } _write() { + _kinit execute_robot_test ${container} -N "xcompat-cluster-${cluster_version}-client-${client_version}-write" -v SUFFIX:${client_version} compatibility/write.robot } _read() { + _kinit local data_version="$1" execute_robot_test ${container} -N "xcompat-cluster-${cluster_version}-client-${client_version}-read-${data_version}" -v SUFFIX:${data_version} compatibility/read.robot } +test_bucket_encryption() { + _kinit + execute_robot_test ${container} -N "xcompat-cluster-${cluster_version}-client-${client_version}" -v SUFFIX:${client_version} security/bucket-encryption.robot +} + test_cross_compatibility() { echo "Starting cluster with COMPOSE_FILE=${COMPOSE_FILE}" OZONE_KEEP_RESULTS=true start_docker_env + execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME} + new_client test_bucket_encryption + + container=scm _kinit execute_command_in_container scm ozone freon ockg -n1 -t1 -p warmup new_client _write new_client _read ${current_version} @@ -65,6 +85,8 @@ test_cross_compatibility() { for client_version in "$@"; do client="old_client_${client_version//./_}" + old_client test_bucket_encryption + old_client _write old_client _read ${client_version} @@ -79,7 +101,8 @@ test_ec_cross_compatibility() { echo "Running Erasure Coded storage backward compatibility tests." # local cluster_versions_with_ec="1.3.0 1.4.0 ${current_version}" local cluster_versions_with_ec="${current_version}" # until HDDS-11334 - local non_ec_client_versions="1.0.0 1.1.0 1.2.1" + # TODO: debug acceptance test failures for client versions 1.0.0 on secure clusters + local non_ec_client_versions="1.1.0 1.2.1" for cluster_version in ${cluster_versions_with_ec}; do export COMPOSE_FILE=new-cluster.yaml:clients.yaml cluster_version=${cluster_version} @@ -102,12 +125,14 @@ test_ec_cross_compatibility() { local prefix=$(LC_CTYPE=C tr -dc '[:alnum:]' < /dev/urandom | head -c 5 | tr '[:upper:]' '[:lower:]') OZONE_DIR=/opt/hadoop + new_client _kinit execute_robot_test new_client --include setup-ec-data -N "xcompat-cluster-${cluster_version}-setup-data" -v prefix:"${prefix}" ec/backward-compat.robot OZONE_DIR=/opt/ozone for client_version in ${non_ec_client_versions}; do client="old_client_${client_version//./_}" unset OUTPUT_PATH + container="${client}" _kinit execute_robot_test "${client}" --include test-ec-compat -N "xcompat-cluster-${cluster_version}-client-${client_version}-read-${cluster_version}" -v prefix:"${prefix}" ec/backward-compat.robot done diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index dd54b7205ed..f71dc44fec5 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -150,7 +150,6 @@ enum Type { RenameSnapshot = 131; ListOpenFiles = 132; QuotaRepair = 133; - GetServerDefaults = 134; GetQuotaRepairStatus = 135; StartQuotaRepair = 136; SnapshotMoveTableKeys = 137; @@ -293,7 +292,6 @@ message OMRequest { optional RenameSnapshotRequest RenameSnapshotRequest = 129; optional ListOpenFilesRequest ListOpenFilesRequest = 130; optional QuotaRepairRequest QuotaRepairRequest = 131; - optional ServerDefaultsRequest ServerDefaultsRequest = 132; optional GetQuotaRepairStatusRequest GetQuotaRepairStatusRequest = 133; optional StartQuotaRepairRequest StartQuotaRepairRequest = 134; optional SnapshotMoveTableKeysRequest SnapshotMoveTableKeysRequest = 135; @@ -424,7 +422,6 @@ message OMResponse { optional RenameSnapshotResponse RenameSnapshotResponse = 132; optional ListOpenFilesResponse ListOpenFilesResponse = 133; optional QuotaRepairResponse QuotaRepairResponse = 134; - optional ServerDefaultsResponse ServerDefaultsResponse = 135; optional GetQuotaRepairStatusResponse GetQuotaRepairStatusResponse = 136; optional StartQuotaRepairResponse StartQuotaRepairResponse = 137; } @@ -1640,6 +1637,7 @@ message ServiceInfo { repeated ServicePort servicePorts = 3; optional OMRoleInfo omRole = 4; optional int32 OMVersion = 5 [default = 0]; + optional FsServerDefaultsProto serverDefaults = 6; } message MultipartInfoInitiateRequest { @@ -2239,17 +2237,10 @@ message BucketQuotaCount { message QuotaRepairResponse { } -message ServerDefaultsRequest { -} - message FsServerDefaultsProto { optional string keyProviderUri = 1; } -message ServerDefaultsResponse { - required FsServerDefaultsProto serverDefaults = 1; -} - message GetQuotaRepairStatusRequest { } message GetQuotaRepairStatusResponse { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index afbe980eeb6..c4322309173 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -437,7 +437,6 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl private List ratisReporterList = null; private KeyProviderCryptoExtension kmsProvider; - private OzoneFsServerDefaults serverDefaults; private final OMLayoutVersionManager versionManager; private final ReplicationConfigValidator replicationConfigValidator; @@ -655,14 +654,6 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) kmsProvider = null; LOG.error("Fail to create Key Provider"); } - Configuration hadoopConfig = - LegacyHadoopConfigurationSource.asHadoopConfiguration(configuration); - URI keyProviderUri = KMSUtil.getKeyProviderUri( - hadoopConfig, - CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH); - String keyProviderUriStr = - (keyProviderUri != null) ? keyProviderUri.toString() : null; - serverDefaults = new OzoneFsServerDefaults(keyProviderUriStr); if (secConfig.isSecurityEnabled()) { omComponent = OM_DAEMON + "-" + omId; HddsProtos.OzoneManagerDetailsProto omInfo = @@ -3140,6 +3131,15 @@ public List getServiceList() throws IOException { .setType(ServicePort.Type.RPC) .setValue(omRpcAddress.getPort()) .build()); + Configuration hadoopConfig = + LegacyHadoopConfigurationSource.asHadoopConfiguration(configuration); + URI keyProviderUri = KMSUtil.getKeyProviderUri( + hadoopConfig, + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH); + String keyProviderUriStr = + (keyProviderUri != null) ? keyProviderUri.toString() : null; + omServiceInfoBuilder.setServerDefaults( + new OzoneFsServerDefaults(keyProviderUriStr)); if (httpServer != null && httpServer.getHttpAddress() != null) { omServiceInfoBuilder.addServicePort(ServicePort.newBuilder() @@ -4749,11 +4749,6 @@ public boolean setSafeMode(SafeModeAction action, boolean isChecked) } } - @Override - public OzoneFsServerDefaults getServerDefaults() { - return serverDefaults; - } - @Override public String getQuotaRepairStatus() throws IOException { checkAdminUserPrivilege("quota repair status"); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 576fac48c73..5682b040e85 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -130,7 +130,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSafeModeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetSafeModeResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3VolumeContextResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServerDefaultsResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffResponse; @@ -376,12 +375,6 @@ public OMResponse handleReadRequest(OMRequest request) { getSnapshotInfo(request.getSnapshotInfoRequest()); responseBuilder.setSnapshotInfoResponse(snapshotInfoResponse); break; - case GetServerDefaults: - responseBuilder.setServerDefaultsResponse( - ServerDefaultsResponse.newBuilder() - .setServerDefaults(impl.getServerDefaults().getProtobuf()) - .build()); - break; case GetQuotaRepairStatus: OzoneManagerProtocolProtos.GetQuotaRepairStatusResponse quotaRepairStatusRsp = getQuotaRepairStatus(request.getGetQuotaRepairStatusRequest());