Merge branch '2.1'

pom.xml

@@ -269,7 +269,6 @@
         <!-- This was added to ensure project only uses public API -->
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-checkstyle-plugin</artifactId>
-        <version>3.3.0</version>
         <dependencies>
           <dependency>
             <groupId>com.puppycrawl.tools</groupId>

src/main/java/org/apache/accumulo/testing/KerberosHelper.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.testing;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.security.tokens.DelegationToken;
+import org.apache.accumulo.core.conf.ClientProperty;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class KerberosHelper {
+  private static final Logger log = LoggerFactory.getLogger(KerberosHelper.class);
+
+  public static void saslLogin(Properties clientProps, Configuration conf) {
+    if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) {
+      conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+      UserGroupInformation.setConfiguration(conf);
+      try {
+        UserGroupInformation.loginUserFromKeytab(
+            ClientProperty.AUTH_PRINCIPAL.getValue(clientProps),
+            ClientProperty.AUTH_TOKEN.getValue(clientProps));
+      } catch (IOException e) {
+        log.warn("Sasl login failed", e);
+      }
+    }
+  }
+
+  public static Properties configDelegationToken(Properties clientProps)
+      throws AccumuloException, AccumuloSecurityException {
+    if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) {
+      AccumuloClient client = Accumulo.newClient().from(clientProps).build();
+      DelegationToken dt = client.securityOperations().getDelegationToken(null);
+      clientProps = Accumulo.newClientProperties().from(clientProps)
+          .as(ClientProperty.AUTH_PRINCIPAL.getValue(clientProps), dt).build();
+      client.close();
+    }
+    return clientProps;
+  }
+
+  public static Properties configDelegationToken(TestEnv env)
+      throws AccumuloException, AccumuloSecurityException {
+    Properties clientProps = env.getClientProps();
+    if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) {
+      AccumuloClient client = env.getAccumuloClient();
+      clientProps = configDelegationToken(clientProps, client);
+      client.close();
+    }
+    return clientProps;
+  }
+
+  public static Properties configDelegationToken(Properties clientProps, AccumuloClient client)
+      throws AccumuloException, AccumuloSecurityException {
+    if (Boolean.parseBoolean(ClientProperty.SASL_ENABLED.getValue(clientProps))) {
+      DelegationToken dt = client.securityOperations().getDelegationToken(null);
+      clientProps = Accumulo.newClientProperties().from(clientProps)
+          .as(ClientProperty.AUTH_PRINCIPAL.getValue(clientProps), dt).build();
+    }
+    return clientProps;
+  }
+}

src/main/java/org/apache/accumulo/testing/TestEnv.java

@@ -175,6 +175,7 @@ public String getYarnResourceManager() {
    */
   public synchronized AccumuloClient getAccumuloClient() {
     if (client == null) {
+      KerberosHelper.saslLogin(clientProps, getHadoopConfiguration());
       client = Accumulo.newClient().from(clientProps).build();
     }
     return client;

src/main/java/org/apache/accumulo/testing/continuous/ContinuousMoru.java

@@ -21,6 +21,7 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import java.io.IOException;
+import java.util.Properties;
 import java.util.Random;
 import java.util.Set;
 import java.util.UUID;
@@ -32,6 +33,7 @@
 import org.apache.accumulo.core.security.ColumnVisibility;
 import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat;
 import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.testing.KerberosHelper;
 import org.apache.accumulo.testing.TestProps;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
@@ -124,14 +126,16 @@ public int run(String[] args) throws Exception {
       Set<Range> ranges = env.getAccumuloClient().tableOperations()
           .splitRangeByTablets(env.getAccumuloTableName(), new Range(), maxMaps);
 
-      AccumuloInputFormat.configure().clientProperties(env.getClientProps())
+      Properties clientProps = KerberosHelper.configDelegationToken(env);
+
+      AccumuloInputFormat.configure().clientProperties(clientProps)
           .table(env.getAccumuloTableName()).ranges(ranges).autoAdjustRanges(false).store(job);
 
       job.setMapperClass(CMapper.class);
       job.setNumReduceTasks(0);
       job.setOutputFormatClass(AccumuloOutputFormat.class);
 
-      AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true)
+      AccumuloOutputFormat.configure().clientProperties(clientProps).createTables(true)
           .defaultTable(env.getAccumuloTableName()).store(job);
 
       Configuration conf = job.getConfiguration();

src/main/java/org/apache/accumulo/testing/mapreduce/RowHash.java

@@ -30,6 +30,7 @@
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat;
 import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.testing.KerberosHelper;
 import org.apache.accumulo.testing.TestEnv;
 import org.apache.accumulo.testing.TestProps;
 import org.apache.hadoop.conf.Configured;
@@ -78,10 +79,12 @@ public int run(String[] args) throws Exception {
       String inputTable = props.getProperty(TestProps.ROWHASH_INPUT_TABLE);
       String outputTable = props.getProperty(TestProps.ROWHASH_OUTPUT_TABLE);
 
-      AccumuloInputFormat.configure().clientProperties(env.getClientProps()).table(inputTable)
+      Properties clientProps = KerberosHelper.configDelegationToken(env);
+
+      AccumuloInputFormat.configure().clientProperties(clientProps).table(inputTable)
           .fetchColumns(cols).store(job);
-      AccumuloOutputFormat.configure().clientProperties(env.getClientProps())
-          .defaultTable(outputTable).createTables(true).store(job);
+      AccumuloOutputFormat.configure().clientProperties(clientProps).defaultTable(outputTable)
+          .createTables(true).store(job);
 
       job.getConfiguration().set("mapreduce.job.classloader", "true");
       job.setMapperClass(HashDataMapper.class);

src/main/java/org/apache/accumulo/testing/mapreduce/TeraSortIngest.java

@@ -30,6 +30,7 @@
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.testing.KerberosHelper;
 import org.apache.accumulo.testing.TestEnv;
 import org.apache.accumulo.testing.TestProps;
 import org.apache.hadoop.conf.Configuration;
@@ -374,7 +375,9 @@ public int run(String[] args) throws Exception {
       String tableName = testProps.getProperty(TestProps.TERASORT_TABLE);
       Objects.requireNonNull(tableName);
 
-      AccumuloOutputFormat.configure().clientProperties(env.getClientProps()).createTables(true)
+      Properties clientProps = KerberosHelper.configDelegationToken(env);
+
+      AccumuloOutputFormat.configure().clientProperties(clientProps).createTables(true)
           .defaultTable(tableName).store(job);
 
       Configuration conf = job.getConfiguration();

src/main/java/org/apache/accumulo/testing/randomwalk/multitable/CopyTool.java

@@ -28,6 +28,8 @@
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat;
 import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.testing.KerberosHelper;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
@@ -58,10 +60,13 @@ public int run(String[] args) throws Exception {
       return 1;
     }
 
-    Properties props = Accumulo.newClientProperties().from(args[0]).build();
     job.setInputFormatClass(AccumuloInputFormat.class);
 
-    AccumuloInputFormat.configure().clientProperties(props).table(args[1])
+    Properties clientProps = Accumulo.newClientProperties().from(args[0]).build();
+    KerberosHelper.saslLogin(clientProps, new Configuration(false));
+    clientProps = KerberosHelper.configDelegationToken(clientProps);
+
+    AccumuloInputFormat.configure().clientProperties(clientProps).table(args[1])
         .auths(Authorizations.EMPTY).store(job);
 
     job.setMapperClass(SeqMapClass.class);
@@ -71,7 +76,7 @@ public int run(String[] args) throws Exception {
     job.getConfiguration().set("mapreduce.job.classloader", "true");
 
     job.setOutputFormatClass(AccumuloOutputFormat.class);
-    AccumuloOutputFormat.configure().clientProperties(props).createTables(true)
+    AccumuloOutputFormat.configure().clientProperties(clientProps).createTables(true)
         .defaultTable(args[2]).store(job);
 
     job.waitForCompletion(true);

src/main/java/org/apache/accumulo/testing/randomwalk/sequential/MapRedVerifyTool.java

@@ -28,6 +28,8 @@
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat;
 import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.testing.KerberosHelper;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.NullWritable;
@@ -91,10 +93,12 @@ public int run(String[] args) throws Exception {
       return 1;
     }
 
-    Properties props = Accumulo.newClientProperties().from(args[0]).build();
+    Properties clientProps = Accumulo.newClientProperties().from(args[0]).build();
+    KerberosHelper.saslLogin(clientProps, new Configuration(false));
+    clientProps = KerberosHelper.configDelegationToken(clientProps);
 
-    AccumuloInputFormat.configure().clientProperties(props).table(args[1]).store(job);
-    AccumuloOutputFormat.configure().clientProperties(props).defaultTable(args[2])
+    AccumuloInputFormat.configure().clientProperties(clientProps).table(args[1]).store(job);
+    AccumuloOutputFormat.configure().clientProperties(clientProps).defaultTable(args[2])
         .createTables(true).store(job);
 
     job.setInputFormatClass(AccumuloInputFormat.class);