diff --git a/pom.xml b/pom.xml index 7c2a15a5..e53da23c 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,6 @@ org.apache.maven.plugins maven-checkstyle-plugin - 3.3.0 com.puppycrawl.tools diff --git a/src/main/java/org/apache/accumulo/testing/KerberosHelper.java b/src/main/java/org/apache/accumulo/testing/KerberosHelper.java new file mode 100644 index 00000000..4d3bc08f --- /dev/null +++ b/src/main/java/org/apache/accumulo/testing/KerberosHelper.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.testing; + +import java.io.IOException; +import java.util.Properties; + +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.security.tokens.DelegationToken; +import org.apache.accumulo.core.conf.ClientProperty; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class KerberosHelper { + private static final Logger log = LoggerFactory.getLogger(KerberosHelper.class); + + public static void saslLogin(Properties clientProps, Configuration conf) { + if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) { + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(conf); + try { + UserGroupInformation.loginUserFromKeytab( + ClientProperty.AUTH_PRINCIPAL.getValue(clientProps), + ClientProperty.AUTH_TOKEN.getValue(clientProps)); + } catch (IOException e) { + log.warn("Sasl login failed", e); + } + } + } + + public static Properties configDelegationToken(Properties clientProps) + throws AccumuloException, AccumuloSecurityException { + if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) { + AccumuloClient client = Accumulo.newClient().from(clientProps).build(); + DelegationToken dt = client.securityOperations().getDelegationToken(null); + clientProps = Accumulo.newClientProperties().from(clientProps) + .as(ClientProperty.AUTH_PRINCIPAL.getValue(clientProps), dt).build(); + client.close(); + } + return clientProps; + } + + public static Properties configDelegationToken(TestEnv env) + throws AccumuloException, AccumuloSecurityException { + Properties clientProps = env.getClientProps(); + if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) { + AccumuloClient client = env.getAccumuloClient(); + clientProps = configDelegationToken(clientProps, client); + client.close(); + } + return clientProps; + } + + public static Properties configDelegationToken(Properties clientProps, AccumuloClient client) + throws AccumuloException, AccumuloSecurityException { + if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) { + DelegationToken dt = client.securityOperations().getDelegationToken(null); + clientProps = Accumulo.newClientProperties().from(clientProps) + .as(ClientProperty.AUTH_PRINCIPAL.getValue(clientProps), dt).build(); + } + return clientProps; + } +} diff --git a/src/main/java/org/apache/accumulo/testing/TestEnv.java b/src/main/java/org/apache/accumulo/testing/TestEnv.java index 929206c5..a7ecf739 100644 --- a/src/main/java/org/apache/accumulo/testing/TestEnv.java +++ b/src/main/java/org/apache/accumulo/testing/TestEnv.java @@ -175,6 +175,7 @@ public String getYarnResourceManager() { */ public synchronized AccumuloClient getAccumuloClient() { if (client == null) { + KerberosHelper.saslLogin(clientProps, getHadoopConfiguration()); client = Accumulo.newClient().from(clientProps).build(); } return client; diff --git a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java index 5c0338fa..285898cc 100644 --- a/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java +++ b/src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java @@ -21,6 +21,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; +import java.util.Properties; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -32,6 +33,7 @@ import org.apache.accumulo.core.security.ColumnVisibility; import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat; import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat; +import org.apache.accumulo.testing.KerberosHelper; import org.apache.accumulo.testing.TestProps; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; @@ -124,14 +126,16 @@ public int run(String[] args) throws Exception { Set ranges = env.getAccumuloClient().tableOperations() .splitRangeByTablets(env.getAccumuloTableName(), new Range(), maxMaps); - AccumuloInputFormat.configure().clientProperties(env.getClientProps()) + Properties clientProps = KerberosHelper.configDelegationToken(env); + + AccumuloInputFormat.configure().clientProperties(clientProps) .table(env.getAccumuloTableName()).ranges(ranges).autoAdjustRanges(false).store(job); job.setMapperClass(CMapper.class); job.setNumReduceTasks(0); job.setOutputFormatClass(AccumuloOutputFormat.class); - AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true) + AccumuloOutputFormat.configure().clientProperties(clientProps).createTables(true) .defaultTable(env.getAccumuloTableName()).store(job); Configuration conf = job.getConfiguration(); diff --git a/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java b/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java index 9eb334ab..b99f4985 100644 --- a/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java +++ b/src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java @@ -30,6 +30,7 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat; import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat; +import org.apache.accumulo.testing.KerberosHelper; import org.apache.accumulo.testing.TestEnv; import org.apache.accumulo.testing.TestProps; import org.apache.hadoop.conf.Configured; @@ -78,10 +79,12 @@ public int run(String[] args) throws Exception { String inputTable = props.getProperty(TestProps.ROWHASH_INPUT_TABLE); String outputTable = props.getProperty(TestProps.ROWHASH_OUTPUT_TABLE); - AccumuloInputFormat.configure().clientProperties(env.getClientProps()).table(inputTable) + Properties clientProps = KerberosHelper.configDelegationToken(env); + + AccumuloInputFormat.configure().clientProperties(clientProps).table(inputTable) .fetchColumns(cols).store(job); - AccumuloOutputFormat.configure().clientProperties(env.getClientProps()) - .defaultTable(outputTable).createTables(true).store(job); + AccumuloOutputFormat.configure().clientProperties(clientProps).defaultTable(outputTable) + .createTables(true).store(job); job.getConfiguration().set("mapreduce.job.classloader", "true"); job.setMapperClass(HashDataMapper.class); diff --git a/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java b/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java index 91522b84..e40954ee 100644 --- a/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java +++ b/src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java @@ -30,6 +30,7 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat; +import org.apache.accumulo.testing.KerberosHelper; import org.apache.accumulo.testing.TestEnv; import org.apache.accumulo.testing.TestProps; import org.apache.hadoop.conf.Configuration; @@ -374,7 +375,9 @@ public int run(String[] args) throws Exception { String tableName = testProps.getProperty(TestProps.TERASORT_TABLE); Objects.requireNonNull(tableName); - AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true) + Properties clientProps = KerberosHelper.configDelegationToken(env); + + AccumuloOutputFormat.configure().clientProperties(clientProps).createTables(true) .defaultTable(tableName).store(job); Configuration conf = job.getConfiguration(); diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTool.java b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTool.java index 8ec8e51b..c081958f 100644 --- a/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTool.java +++ b/src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTool.java @@ -28,6 +28,8 @@ import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat; import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat; +import org.apache.accumulo.testing.KerberosHelper; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; @@ -58,10 +60,13 @@ public int run(String[] args) throws Exception { return 1; } - Properties props = Accumulo.newClientProperties().from(args[0]).build(); job.setInputFormatClass(AccumuloInputFormat.class); - AccumuloInputFormat.configure().clientProperties(props).table(args[1]) + Properties clientProps = Accumulo.newClientProperties().from(args[0]).build(); + KerberosHelper.saslLogin(clientProps, new Configuration(false)); + clientProps = KerberosHelper.configDelegationToken(clientProps); + + AccumuloInputFormat.configure().clientProperties(clientProps).table(args[1]) .auths(Authorizations.EMPTY).store(job); job.setMapperClass(SeqMapClass.class); @@ -71,7 +76,7 @@ public int run(String[] args) throws Exception { job.getConfiguration().set("mapreduce.job.classloader", "true"); job.setOutputFormatClass(AccumuloOutputFormat.class); - AccumuloOutputFormat.configure().clientProperties(props).createTables(true) + AccumuloOutputFormat.configure().clientProperties(clientProps).createTables(true) .defaultTable(args[2]).store(job); job.waitForCompletion(true); diff --git a/src/main/java/org/apache/accumulo/testing/randomwalk/sequential/MapRedVerifyTool.java b/src/main/java/org/apache/accumulo/testing/randomwalk/sequential/MapRedVerifyTool.java index a94826d0..310b1034 100644 --- a/src/main/java/org/apache/accumulo/testing/randomwalk/sequential/MapRedVerifyTool.java +++ b/src/main/java/org/apache/accumulo/testing/randomwalk/sequential/MapRedVerifyTool.java @@ -28,6 +28,8 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat; import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat; +import org.apache.accumulo.testing.KerberosHelper; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.NullWritable; @@ -91,10 +93,12 @@ public int run(String[] args) throws Exception { return 1; } - Properties props = Accumulo.newClientProperties().from(args[0]).build(); + Properties clientProps = Accumulo.newClientProperties().from(args[0]).build(); + KerberosHelper.saslLogin(clientProps, new Configuration(false)); + clientProps = KerberosHelper.configDelegationToken(clientProps); - AccumuloInputFormat.configure().clientProperties(props).table(args[1]).store(job); - AccumuloOutputFormat.configure().clientProperties(props).defaultTable(args[2]) + AccumuloInputFormat.configure().clientProperties(clientProps).table(args[1]).store(job); + AccumuloOutputFormat.configure().clientProperties(clientProps).defaultTable(args[2]) .createTables(true).store(job); job.setInputFormatClass(AccumuloInputFormat.class);