Skip to content

Commit 1f7b0bf

Browse files
ebogiebogi
committed
SQOOP-3391: Test storing AWS credentials in Hadoop CredentialProvider during import
(Boglarka Egyed)
1 parent 5dd8c8a commit 1f7b0bf

10 files changed

+238
-0
lines changed

‎build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ dependencies {
141141
testCompile group: 'junit', name: 'junit', version: junitVersion
142142
testCompile group: 'org.assertj', name: 'assertj-core', version: assertjVersion
143143
testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoallVersion
144+
testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: systemRulesVersion
144145
testCompile group: 'org.apache.zookeeper', name: 'zookeeper', version: zookeeperVersion, ext: 'jar'
145146
}
146147

‎build.xml

+4
Original file line numberDiff line numberDiff line change
@@ -745,6 +745,10 @@
745745
<copy todir="${test.build.extraconf}/oraoop">
746746
<fileset dir="${test.dir}/oraoop"/>
747747
</copy>
748+
<copy file="${basedir}/conf/password-file.txt"
749+
todir="${test.build.extraconf}" />
750+
<copy file="${basedir}/conf/wrong-password-file.txt"
751+
todir="${test.build.extraconf}" />
748752
<junit
749753
printsummary="yes" showoutput="${test.output}"
750754
haltonfailure="no" fork="yes" maxmemory="5120m"

‎conf/password-file.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
credProviderPwd

‎conf/wrong-password-file.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
wrongCredProviderPwd

‎gradle.properties

+1
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ commonsnetVersion=3.1
4848
log4jVersion=1.2.16
4949
junitVersion=4.12
5050
mockitoallVersion=1.9.5
51+
systemRulesVersion=1.17.0
5152
assertjVersion=2.8.0
5253

5354
checkstyleVersion=5.5

‎ivy.xml

+2
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,8 @@ under the License.
105105
conf="test->default"/>
106106
<dependency org="org.mockito" name="mockito-all"
107107
rev="${mockito-all.version}" conf="test->default"/>
108+
<dependency org="com.github.stefanbirkner" name="system-rules"
109+
rev="${system-rules.version}" conf="test->default"/>
108110
<!-- We're only using H2 for tests as it supports stored
109111
procedures; once we move to HSQLDB 2.x we can drop
110112
this -->

‎ivy/libraries.properties

+1
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ ivy.version=2.3.0
4141
junit.version=4.12
4242
assertj.version=2.8.0
4343
mockito-all.version=1.9.5
44+
system-rules.version=1.17.0
4445

4546
h2.version=1.3.170
4647

‎src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java

+2
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ public class CredentialProviderHelper {
8585
// Should track what is in CredentialProvider class.
8686
public static final String CREDENTIAL_PROVIDER_PATH =
8787
"hadoop.security.credential.provider.path";
88+
public static final String CREDENTIAL_PROVIDER_PASSWORD_FILE =
89+
"hadoop.security.credstore.java-keystore-provider.password-file";
8890

8991
public static boolean isProviderAvailable() {
9092

‎src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java

+213
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.sqoop.s3;
20+
21+
import org.apache.commons.logging.Log;
22+
import org.apache.commons.logging.LogFactory;
23+
import org.apache.hadoop.conf.Configuration;
24+
import org.apache.hadoop.fs.FileSystem;
25+
import org.apache.hadoop.fs.s3a.Constants;
26+
import org.apache.hadoop.security.alias.CredentialShell;
27+
import org.apache.hadoop.util.ToolRunner;
28+
import org.apache.sqoop.testutil.ArgumentArrayBuilder;
29+
import org.apache.sqoop.testutil.DefaultS3CredentialGenerator;
30+
import org.apache.sqoop.testutil.ImportJobTestCase;
31+
import org.apache.sqoop.testutil.S3CredentialGenerator;
32+
import org.apache.sqoop.testutil.S3TestUtils;
33+
import org.apache.sqoop.testutil.TextFileTestUtils;
34+
import org.apache.sqoop.util.password.CredentialProviderHelper;
35+
import org.junit.After;
36+
import org.junit.AfterClass;
37+
import org.junit.Before;
38+
import org.junit.BeforeClass;
39+
import org.junit.ClassRule;
40+
import org.junit.Rule;
41+
import org.junit.Test;
42+
import org.junit.contrib.java.lang.system.EnvironmentVariables;
43+
import org.junit.rules.ExpectedException;
44+
45+
import java.io.File;
46+
import java.io.IOException;
47+
import java.nio.file.Files;
48+
49+
import static junit.framework.TestCase.fail;
50+
51+
public class TestS3ImportWithHadoopCredProvider extends ImportJobTestCase {
52+
public static final Log LOG = LogFactory.getLog(
53+
TestS3ImportWithHadoopCredProvider.class.getName());
54+
55+
private static S3CredentialGenerator s3CredentialGenerator;
56+
57+
private static String providerPathDefault;
58+
private static String providerPathEnv;
59+
private static String providerPathPwdFile;
60+
61+
@ClassRule
62+
public static final EnvironmentVariables environmentVariables
63+
= new EnvironmentVariables();
64+
private static File providerFileDefault;
65+
private static File providerFileEnvPwd;
66+
private static File providerFilePwdFile;
67+
68+
private FileSystem s3Client;
69+
70+
private static final String PASSWORD_FILE_NAME = "password-file.txt";
71+
private static final String HADOOP_CREDSTORE_PASSWORD_ENV_NAME = "HADOOP_CREDSTORE_PASSWORD";
72+
73+
@Rule
74+
public ExpectedException thrown = ExpectedException.none();
75+
76+
@BeforeClass
77+
public static void setupS3Credentials() throws Exception {
78+
String generatorCommand = S3TestUtils.getGeneratorCommand();
79+
if (generatorCommand != null) {
80+
s3CredentialGenerator = new DefaultS3CredentialGenerator(generatorCommand);
81+
}
82+
generateTempProviderFileNames();
83+
fillCredentialProviderDefault();
84+
fillCredentialProviderPwdFile();
85+
fillCredentialProviderEnv();
86+
}
87+
88+
@Before
89+
public void setup() throws IOException {
90+
S3TestUtils.runTestCaseOnlyIfS3CredentialsAreSet(s3CredentialGenerator);
91+
super.setUp();
92+
S3TestUtils.createTestTableFromInputData(this);
93+
s3Client = S3TestUtils.setupS3ImportTestCase(s3CredentialGenerator);
94+
environmentVariables.clear(HADOOP_CREDSTORE_PASSWORD_ENV_NAME);
95+
}
96+
97+
@After
98+
public void cleanUpTargetDir() {
99+
S3TestUtils.tearDownS3ImportTestCase(s3Client);
100+
super.tearDown();
101+
}
102+
103+
@AfterClass
104+
public static void deleteTemporaryCredFiles() {
105+
providerFileDefault.deleteOnExit();
106+
providerFileEnvPwd.deleteOnExit();
107+
providerFilePwdFile.deleteOnExit();
108+
}
109+
110+
@Test
111+
public void testCredentialProviderDefaultSucceeds() throws Exception {
112+
runImport(getArgs(providerPathDefault,false, null));
113+
TextFileTestUtils.verify(S3TestUtils.getExpectedTextOutput(), s3Client, S3TestUtils.getTargetDirPath());
114+
}
115+
116+
@Test
117+
public void testCredentialProviderEnvSucceeds() throws Exception {
118+
setHadoopCredStorePwdEnvVar();
119+
runImport(getArgs(providerPathEnv,false, null));
120+
TextFileTestUtils.verify(S3TestUtils.getExpectedTextOutput(), s3Client, S3TestUtils.getTargetDirPath());
121+
}
122+
123+
@Test
124+
public void testCredentialProviderPwdFileSucceeds() throws Exception {
125+
runImport(getArgs(providerPathPwdFile,true, PASSWORD_FILE_NAME));
126+
TextFileTestUtils.verify(S3TestUtils.getExpectedTextOutput(), s3Client, S3TestUtils.getTargetDirPath());
127+
}
128+
129+
@Test
130+
public void testCredentialProviderWithNoProviderPathFails() throws Exception {
131+
thrown.expect(IOException.class);
132+
runImport(getArgs(null,false, null));
133+
}
134+
135+
@Test
136+
public void testCredentialProviderWithNoEnvFails() throws Exception {
137+
thrown.expect(IOException.class);
138+
runImport(getArgs(providerPathEnv,false, null));
139+
}
140+
141+
@Test
142+
public void testCredentialProviderWithWrongPwdFileFails() throws Exception {
143+
thrown.expect(IOException.class);
144+
runImport(getArgs(providerPathPwdFile,true, "wrong-password-file.txt"));
145+
}
146+
147+
@Test
148+
public void testCredentialProviderWithNoPwdFileFails() throws Exception {
149+
thrown.expect(IOException.class);
150+
runImport(getArgs(providerPathPwdFile,true, null));
151+
}
152+
153+
private String[] getArgs(String providerPath, boolean withPwdFile, String pwdFile) {
154+
ArgumentArrayBuilder builder = S3TestUtils.getArgumentArrayBuilderForHadoopCredProviderS3UnitTests(this);
155+
156+
builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH, providerPath);
157+
if (withPwdFile) {
158+
builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PASSWORD_FILE, pwdFile);
159+
}
160+
return builder.build();
161+
}
162+
163+
private static void fillCredentialProviderDefault() throws Exception {
164+
fillCredentialProvider(new Configuration(), providerPathDefault);
165+
}
166+
167+
private static void fillCredentialProviderEnv() throws Exception {
168+
setHadoopCredStorePwdEnvVar();
169+
fillCredentialProvider(new Configuration(), providerPathEnv);
170+
}
171+
172+
private static void fillCredentialProviderPwdFile() throws Exception {
173+
Configuration conf = new Configuration();
174+
conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PASSWORD_FILE, PASSWORD_FILE_NAME);
175+
fillCredentialProvider(conf, providerPathPwdFile);
176+
}
177+
178+
private static void generateTempProviderFileNames() throws IOException {
179+
providerFileDefault = Files.createTempFile("test-default-pwd-", ".jceks").toFile();
180+
boolean deleted = providerFileDefault.delete();
181+
providerFileEnvPwd = Files.createTempFile("test-env-pwd-", ".jceks").toFile();
182+
deleted &= providerFileEnvPwd.delete();
183+
providerFilePwdFile = Files.createTempFile("test-file-pwd-", ".jceks").toFile();
184+
deleted &= providerFilePwdFile.delete();
185+
if (!deleted) {
186+
fail("Could not delete temporary provider files");
187+
}
188+
providerPathDefault = "jceks://file/" + providerFileDefault.getAbsolutePath();
189+
providerPathEnv = "jceks://file/" + providerFileEnvPwd.getAbsolutePath();
190+
providerPathPwdFile = "jceks://file/" + providerFilePwdFile.getAbsolutePath();
191+
}
192+
193+
private static void runCredentialProviderCreateCommand(String command, Configuration conf) throws Exception {
194+
ToolRunner.run(conf, new CredentialShell(), command.split(" "));
195+
}
196+
197+
private static String getCreateCommand(String credentialKey, String credentialValue, String providerPath) {
198+
return "create " + credentialKey + " -value " + credentialValue + " -provider " + providerPath;
199+
}
200+
201+
private static void fillCredentialProvider(Configuration conf, String providerPath) throws Exception {
202+
runCredentialProviderCreateCommand(getCreateCommand(Constants.ACCESS_KEY, s3CredentialGenerator.getS3AccessKey(), providerPath), conf);
203+
runCredentialProviderCreateCommand(getCreateCommand(Constants.SECRET_KEY, s3CredentialGenerator.getS3SecretKey(), providerPath), conf);
204+
205+
if (s3CredentialGenerator.getS3SessionToken() != null) {
206+
runCredentialProviderCreateCommand(getCreateCommand(Constants.SESSION_TOKEN, s3CredentialGenerator.getS3SessionToken(), providerPath), conf);
207+
}
208+
}
209+
210+
private static void setHadoopCredStorePwdEnvVar() {
211+
environmentVariables.set(HADOOP_CREDSTORE_PASSWORD_ENV_NAME, "credProviderPwd");
212+
}
213+
}

‎src/test/org/apache/sqoop/testutil/S3TestUtils.java

+12
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,18 @@ public static ArgumentArrayBuilder getArgumentArrayBuilderForS3UnitTests(BaseSqo
178178
.withOption("target-dir", getTargetDirPath().toString());
179179
}
180180

181+
public static ArgumentArrayBuilder getArgumentArrayBuilderForHadoopCredProviderS3UnitTests(BaseSqoopTestCase testCase) {
182+
183+
ArgumentArrayBuilder builder = new ArgumentArrayBuilder();
184+
return builder.withCommonHadoopFlags()
185+
.withProperty("fs.s3a.impl.disable.cache", "true")
186+
.withProperty(Constants.AWS_CREDENTIALS_PROVIDER, getTemporaryCredentialsProviderClass())
187+
.withOption("connect", testCase.getConnectString())
188+
.withOption("num-mappers", "1")
189+
.withOption("table", testCase.getTableName())
190+
.withOption("target-dir", getTargetDirPath().toString());
191+
}
192+
181193
public static ArgumentArrayBuilder getArgumentArrayBuilderForS3UnitTestsWithFileFormatOption(BaseSqoopTestCase testCase,
182194
S3CredentialGenerator s3CredentialGenerator,
183195
String fileFormat) {

0 commit comments

Comments
 (0)