From 9ca32e9b526689dc94673beba69a01c62442b50f Mon Sep 17 00:00:00 2001 From: zhaozihao Date: Mon, 14 Aug 2023 11:51:04 +0800 Subject: [PATCH 1/2] Add UT Tests --- .../underfs/gcs/GCSUnderFileSystem.java | 2 +- .../gcs/GCSUnderFileSystemFactory.java | 42 ++-- .../underfs/gcs/GCSOutputStreamTest.java | 207 ++++++++++++++++++ .../underfs/gcs/GCSPositionReaderTest.java | 84 +++++++ .../gcs/GCSUnderFileSystemFactoryTest.java | 88 +++++++- .../underfs/gcs/GCSUnderFileSystemTest.java | 150 ++++++++++++- .../underfs/obs/OBSUnderFileSystem.java | 2 +- .../obs/OBSUnderFileSystemFactory.java | 3 +- .../underfs/obs/OBSInputStreamTest.java | 48 ++-- .../underfs/obs/OBSPositionReaderTest.java | 84 +++++++ .../obs/OBSUnderFileSystemFactoryTest.java | 87 +++++++- .../underfs/obs/OBSUnderFileSystemTest.java | 175 ++++++++++++++- .../underfs/oss/OSSUnderFileSystem.java | 2 +- .../oss/OSSUnderFileSystemFactory.java | 3 +- .../underfs/oss/OSSInputStreamTest.java | 44 ++-- .../underfs/oss/OSSPositionReaderTest.java | 84 +++++++ .../oss/OSSUnderFileSystemFactoryTest.java | 87 +++++++- .../underfs/oss/OSSUnderFileSystemTest.java | 183 +++++++++++++++- 18 files changed, 1303 insertions(+), 72 deletions(-) create mode 100644 dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSOutputStreamTest.java create mode 100644 dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSPositionReaderTest.java create mode 100644 dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSPositionReaderTest.java create mode 100644 dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSPositionReaderTest.java diff --git a/dora/underfs/gcs/src/main/java/alluxio/underfs/gcs/GCSUnderFileSystem.java b/dora/underfs/gcs/src/main/java/alluxio/underfs/gcs/GCSUnderFileSystem.java index 55b8deb81ecd..cd5c4dbce75d 100644 --- a/dora/underfs/gcs/src/main/java/alluxio/underfs/gcs/GCSUnderFileSystem.java +++ b/dora/underfs/gcs/src/main/java/alluxio/underfs/gcs/GCSUnderFileSystem.java @@ -230,7 +230,7 @@ protected ObjectListingChunk getObjectListingChunk(String key, boolean recursive } // Get next chunk of listing result. - private StorageObjectsChunk getObjectListingChunk(String key, String delimiter, + protected StorageObjectsChunk getObjectListingChunk(String key, String delimiter, String priorLastKey) { StorageObjectsChunk res; try { diff --git a/dora/underfs/gcs/src/main/java/alluxio/underfs/gcs/GCSUnderFileSystemFactory.java b/dora/underfs/gcs/src/main/java/alluxio/underfs/gcs/GCSUnderFileSystemFactory.java index ff60027538f7..747ad169c0ea 100644 --- a/dora/underfs/gcs/src/main/java/alluxio/underfs/gcs/GCSUnderFileSystemFactory.java +++ b/dora/underfs/gcs/src/main/java/alluxio/underfs/gcs/GCSUnderFileSystemFactory.java @@ -44,27 +44,41 @@ public GCSUnderFileSystemFactory() {} @Override public UnderFileSystem create(String path, UnderFileSystemConfiguration conf) { - Preconditions.checkNotNull(path, "path"); + Preconditions.checkNotNull(path, "Unable to create UnderFileSystem instance:" + + " URI path should not be null"); - if (conf.getInt(PropertyKey.UNDERFS_GCS_VERSION) == GCS_VERSION_TWO) { - try { - return GCSV2UnderFileSystem.createInstance(new AlluxioURI(path), conf); - } catch (IOException e) { - LOG.error("Failed to create GCSV2UnderFileSystem.", e); - throw Throwables.propagate(e); - } - } else { - try { - return GCSUnderFileSystem.createInstance(new AlluxioURI(path), conf); - } catch (ServiceException e) { - LOG.error("Failed to create GCSUnderFileSystem.", e); - throw Throwables.propagate(e); + if (checkGCSCredentials(conf)) { + if (conf.getInt(PropertyKey.UNDERFS_GCS_VERSION) == GCS_VERSION_TWO) { + try { + return GCSV2UnderFileSystem.createInstance(new AlluxioURI(path), conf); + } catch (IOException e) { + LOG.error("Failed to create GCSV2UnderFileSystem.", e); + throw Throwables.propagate(e); + } + } else { + try { + return GCSUnderFileSystem.createInstance(new AlluxioURI(path), conf); + } catch (ServiceException e) { + LOG.error("Failed to create GCSUnderFileSystem.", e); + throw Throwables.propagate(e); + } } } + String err = "GCS credentials or version not available, cannot create GCS Under File System."; + throw Throwables.propagate(new IOException(err)); } @Override public boolean supportsPath(String path) { return path != null && path.startsWith(Constants.HEADER_GCS); } + + /** + * @param conf optional configuration object for the UFS + * @return true if access, secret and endpoint keys are present, false otherwise + */ + private boolean checkGCSCredentials(UnderFileSystemConfiguration conf) { + return conf.isSet(PropertyKey.GCS_ACCESS_KEY) + && conf.isSet(PropertyKey.GCS_SECRET_KEY); + } } diff --git a/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSOutputStreamTest.java b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSOutputStreamTest.java new file mode 100644 index 000000000000..28ed9a9b8b14 --- /dev/null +++ b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSOutputStreamTest.java @@ -0,0 +1,207 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.underfs.gcs; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; + +import alluxio.conf.AlluxioConfiguration; +import alluxio.conf.Configuration; +import alluxio.conf.PropertyKey; + +import org.jclouds.s3.domain.ObjectMetadata; +import org.jets3t.service.ServiceException; +import org.jets3t.service.impl.rest.httpclient.GoogleStorageService; +import org.jets3t.service.model.GSObject; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.DigestOutputStream; + +/** + * Unit tests for the {@link GCSOutputStream}. + */ +@RunWith(PowerMockRunner.class) +public class GCSOutputStreamTest { + private GoogleStorageService mClient; + private File mFile; + private BufferedOutputStream mLocalOutputStream; + private static AlluxioConfiguration sConf = Configuration.global(); + private GSObject mObject; + private String mMd5Hash = "md5 hash"; + + /** + * The exception expected to be thrown. + */ + @Rule + public final ExpectedException mThrown = ExpectedException.none(); + + /** + * Sets the properties and configuration before each test runs. + */ + @Before + public void before() throws Exception { + mClient = Mockito.mock(GoogleStorageService.class); + mObject = Mockito.mock(GSObject.class); + Mockito.when(mObject.getMd5HashAsBase64()).thenReturn(mMd5Hash); + Mockito.when(mClient.putObject(Mockito.anyString(), Mockito.any(GSObject.class))) + .thenReturn(mObject); + mFile = Mockito.mock(File.class); + mLocalOutputStream = Mockito.mock(BufferedOutputStream.class); + } + + /** + * Tests to ensure IOException is thrown if {@link FileOutputStream}() throws an IOException. + */ + @Test + @PrepareForTest(GCSOutputStream.class) + public void testConstructor() throws Exception { + PowerMockito.whenNew(File.class).withArguments(Mockito.anyString()).thenReturn(mFile); + String errorMessage = "protocol doesn't support output"; + PowerMockito.whenNew(FileOutputStream.class).withArguments(mFile) + .thenThrow(new IOException(errorMessage)); + mThrown.expect(IOException.class); + mThrown.expectMessage(errorMessage); + new GCSOutputStream("testBucketName", "testKey", mClient, + sConf.getList(PropertyKey.TMP_DIRS)).close(); + } + + /** + * Tests to ensure {@link GCSOutputStream#write(int)} calls {@link OutputStream#write(int)}. + */ + @Test + @PrepareForTest(GCSOutputStream.class) + public void testWrite1() throws Exception { + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(DigestOutputStream.class)).thenReturn(mLocalOutputStream); + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(FileOutputStream.class)).thenReturn(mLocalOutputStream); + GCSOutputStream stream = new GCSOutputStream("testBucketName", "testKey", mClient, + sConf.getList(PropertyKey.TMP_DIRS)); + stream.write(1); + stream.close(); + assertEquals(mMd5Hash, stream.getContentHash().get()); + Mockito.verify(mLocalOutputStream).write(1); + } + + /** + * Tests to ensure {@link GCSOutputStream#write(byte[], int, int)} calls + * {@link OutputStream#write(byte[], int, int)} . + */ + @Test + @PrepareForTest(GCSOutputStream.class) + public void testWrite2() throws Exception { + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(DigestOutputStream.class)).thenReturn(mLocalOutputStream); + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(FileOutputStream.class)).thenReturn(mLocalOutputStream); + GCSOutputStream stream = new GCSOutputStream("testBucketName", "testKey", mClient, + sConf.getList(PropertyKey.TMP_DIRS)); + byte[] b = new byte[1]; + stream.write(b, 0, 1); + stream.close(); + assertEquals(mMd5Hash, stream.getContentHash().get()); + Mockito.verify(mLocalOutputStream).write(b, 0, 1); + } + + /** + * Tests to ensure {@link GCSOutputStream#write(byte[])} calls {@link OutputStream#write(byte[])}. + */ + @Test + @PrepareForTest(GCSOutputStream.class) + public void testWrite3() throws Exception { + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(DigestOutputStream.class)).thenReturn(mLocalOutputStream); + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(FileOutputStream.class)).thenReturn(mLocalOutputStream); + GCSOutputStream stream = new GCSOutputStream("testBucketName", "testKey", mClient, sConf + .getList(PropertyKey.TMP_DIRS)); + byte[] b = new byte[1]; + stream.write(b); + stream.close(); + assertEquals(mMd5Hash, stream.getContentHash().get()); + Mockito.verify(mLocalOutputStream).write(b, 0, 1); + } + + /** + * Tests to ensure IOException is thrown if + * {@link GoogleStorageService#putObject(String, GSObject)} throws an + * ServiceException. + */ + @Test + @PrepareForTest(GCSOutputStream.class) + public void testCloseError() throws Exception { + String errorMessage = "Invoke the createEmptyObject method error."; + BufferedInputStream inputStream = PowerMockito.mock(BufferedInputStream.class); + PowerMockito.whenNew(BufferedInputStream.class) + .withArguments(any(FileInputStream.class)).thenReturn(inputStream); + PowerMockito + .when(mClient.putObject(Mockito.anyString(), Mockito.any(GSObject.class))) + .thenThrow(new ServiceException(errorMessage)); + GCSOutputStream stream = new GCSOutputStream("testBucketName", "testKey", mClient, sConf + .getList(PropertyKey.TMP_DIRS)); + mThrown.expect(IOException.class); + mThrown.expectMessage(errorMessage); + stream.close(); + assertEquals(mMd5Hash, stream.getContentHash().get()); + } + + /** + * Tests to ensure {@link File#delete()} is called when close the stream. + */ + @Test + @PrepareForTest(GCSOutputStream.class) + public void testCloseSuccess() throws Exception { + PowerMockito.whenNew(File.class).withArguments(Mockito.anyString()).thenReturn(mFile); + FileOutputStream outputStream = PowerMockito.mock(FileOutputStream.class); + PowerMockito.whenNew(FileOutputStream.class).withArguments(mFile).thenReturn(outputStream); + FileInputStream inputStream = PowerMockito.mock(FileInputStream.class); + PowerMockito.whenNew(FileInputStream.class).withArguments(mFile).thenReturn(inputStream); + + GCSOutputStream stream = new GCSOutputStream("testBucketName", "testKey", mClient, sConf + .getList(PropertyKey.TMP_DIRS)); + stream.close(); + assertEquals(mMd5Hash, stream.getContentHash().get()); + Mockito.verify(mFile).delete(); + } + + /** + * Tests to ensure {@link GCSOutputStream#flush()} calls {@link OutputStream#flush()}. + */ + @Test + @PrepareForTest(GCSOutputStream.class) + public void testFlush() throws Exception { + PowerMockito.whenNew(BufferedOutputStream.class) + .withArguments(any(DigestOutputStream.class)).thenReturn(mLocalOutputStream); + GCSOutputStream stream = new GCSOutputStream("testBucketName", "testKey", mClient, sConf + .getList(PropertyKey.TMP_DIRS)); + stream.flush(); + stream.close(); + assertEquals(mMd5Hash, stream.getContentHash().get()); + Mockito.verify(mLocalOutputStream).flush(); + } +} diff --git a/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSPositionReaderTest.java b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSPositionReaderTest.java new file mode 100644 index 000000000000..b98a385e572a --- /dev/null +++ b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSPositionReaderTest.java @@ -0,0 +1,84 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.underfs.gcs; + +import org.jets3t.service.ServiceException; +import org.jets3t.service.impl.rest.httpclient.GoogleStorageService; +import org.jets3t.service.model.GSObject; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; + +public class GCSPositionReaderTest { + /** + * The GCS Position Reader. + */ + private GCSPositionReader mPositionReader; + /** + * The COS Client. + */ + private GoogleStorageService mClient; + /** + * The Bucket Name. + */ + private final String mBucketName = "bucket"; + /** + * The Path (or the Key). + */ + private final String mPath = "path"; + /** + * The File Length. + */ + private final long mFileLength = 100L; + + @Before + public void before() throws Exception { + mClient = Mockito.mock(GoogleStorageService.class); + mPositionReader = new GCSPositionReader(mClient, mBucketName, mPath, mFileLength); + } + + /** + * Test case for {@link GCSPositionReader#openObjectInputStream(long, int)}. + */ + @Test + public void openObjectInputStream() throws Exception { + GSObject object = Mockito.mock(GSObject.class); + BufferedInputStream objectInputStream = Mockito.mock(BufferedInputStream.class); + Mockito.when(mClient.getObject(Mockito.anyString(), Mockito.anyString(), Mockito.any(), + Mockito.any(),Mockito.any(),Mockito.any(),Mockito.any(),Mockito.any())).thenReturn(object); + Mockito.when(object.getDataInputStream()).thenReturn(objectInputStream); + + // test successful open object input stream + long position = 0L; + int bytesToRead = 10; + InputStream inputStream = mPositionReader.openObjectInputStream(position, bytesToRead); + Assert.assertTrue(inputStream instanceof BufferedInputStream); + + // test open object input stream with exception + Mockito.when(mClient.getObject(ArgumentMatchers.anyString(), ArgumentMatchers.anyString())) + .thenThrow(ServiceException.class); + try { + mPositionReader.openObjectInputStream(position, bytesToRead); + } catch (Exception e) { + Assert.assertTrue(e instanceof IOException); + String errorMessage = String + .format("Failed to get object: %s bucket: %s", mPath, mBucketName); + Assert.assertEquals(errorMessage, e.getMessage()); + } + } +} diff --git a/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemFactoryTest.java b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemFactoryTest.java index f0e4cc2d79bb..a15c54fcad8b 100644 --- a/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemFactoryTest.java +++ b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemFactoryTest.java @@ -11,13 +11,21 @@ package alluxio.underfs.gcs; +import alluxio.conf.AlluxioConfiguration; import alluxio.conf.Configuration; +import alluxio.conf.PropertyKey; +import alluxio.underfs.UnderFileSystem; +import alluxio.underfs.UnderFileSystemConfiguration; import alluxio.underfs.UnderFileSystemFactory; import alluxio.underfs.UnderFileSystemFactoryRegistry; +import org.jets3t.service.ServiceException; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertFalse; + /** * Unit tests for the {@link GCSUnderFileSystemFactory}. */ @@ -26,12 +34,84 @@ public final class GCSUnderFileSystemFactoryTest { /** * This test ensures the GCS UFS module correctly accepts paths that begin with gs://. */ + + private final String mPath = "gs://test-bucket/path"; + private final String mAccessKey = "test-access-key"; + private final String mSecretKey = "test-secret-key"; + private AlluxioConfiguration mAlluxioConf; + private UnderFileSystemConfiguration mConf; + private UnderFileSystemFactory mFactory; + + + @Before + public void setUp() { + Configuration.set(PropertyKey.GCS_ACCESS_KEY, mAccessKey); + Configuration.set(PropertyKey.GCS_SECRET_KEY, mSecretKey); + Configuration.set(PropertyKey.UNDERFS_GCS_VERSION, 1); + mAlluxioConf = Configuration.global(); + mConf = UnderFileSystemConfiguration.defaults(mAlluxioConf); + mFactory = UnderFileSystemFactoryRegistry.find(mPath, mConf); + } + @Test public void factory() { - UnderFileSystemFactory factory = UnderFileSystemFactoryRegistry.find("gs://test-bucket/path", - Configuration.global()); - Assert.assertNotNull( - "A UnderFileSystemFactory should exist for gs paths when using this module", factory); + "A UnderFileSystemFactory should exist for gcs paths when using this module", mFactory); } + + /** + * Test case for {@link GCSUnderFileSystemFactory#create(String, UnderFileSystemConfiguration)}. + */ + @Test + public void createInstanceWithNullPath() { + Exception e = Assert.assertThrows(NullPointerException.class, () -> mFactory.create( + null, mConf)); + Assert.assertTrue(e.getMessage().contains("Unable to create UnderFileSystem instance: URI " + + "path should not be null")); + } + + /** + * Test case for {@link GCSUnderFileSystemFactory#create(String, UnderFileSystemConfiguration)}. + */ + @Test + public void createInstanceWithPath() { + UnderFileSystem ufs = mFactory.create(mPath, mConf); + Assert.assertNotNull(ufs); + Assert.assertTrue(ufs instanceof GCSUnderFileSystem); + } + + /** + * Test case for {@link GCSUnderFileSystemFactory#create(String, UnderFileSystemConfiguration)}. + */ + @Test + public void createInstanceWithoutCredentials() { + Configuration.unset(PropertyKey.GCS_ACCESS_KEY); + Configuration.unset(PropertyKey.GCS_SECRET_KEY); + mAlluxioConf = Configuration.global(); + mConf = UnderFileSystemConfiguration.defaults(mAlluxioConf); + + try { + mFactory.create(mPath, mConf); + } catch (Exception e) { + Assert.assertTrue(e instanceof RuntimeException); + Assert.assertTrue(e.getMessage().contains("GCS credentials or version not available, " + + "cannot create GCS Under File System.")); + } + Exception e = Assert.assertThrows(RuntimeException.class, () -> mFactory.create( + mPath, mConf)); + Assert.assertTrue(e.getMessage().contains("GCS credentials or version not available, " + + "cannot create GCS Under File System.")); + } + + /** + * Test case for {@link GCSUnderFileSystemFactory#supportsPath(String)}. + */ + @Test + public void supportsPath() { + Assert.assertTrue(mFactory.supportsPath(mPath)); + assertFalse(mFactory.supportsPath(null)); + assertFalse(mFactory.supportsPath("Invalid_Path")); + assertFalse(mFactory.supportsPath("hdfs://test-bucket/path")); + } + } diff --git a/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemTest.java b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemTest.java index c485c15aaf0a..1f5996ef6917 100644 --- a/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemTest.java +++ b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemTest.java @@ -12,21 +12,34 @@ package alluxio.underfs.gcs; import static org.junit.Assert.assertFalse; +import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import alluxio.AlluxioURI; +import alluxio.Constants; +import alluxio.PositionReader; import alluxio.conf.Configuration; +import alluxio.retry.CountingRetry; +import alluxio.retry.RetryPolicy; import alluxio.underfs.UnderFileSystemConfiguration; import alluxio.underfs.options.DeleteOptions; +import alluxio.underfs.options.OpenOptions; import org.jets3t.service.ServiceException; +import org.jets3t.service.StorageObjectsChunk; import org.jets3t.service.impl.rest.httpclient.GoogleStorageService; +import org.jets3t.service.model.GSObject; +import org.jets3t.service.model.StorageObject; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; /** * Unit tests for the {@link GCSUnderFileSystem}. @@ -38,6 +51,7 @@ public class GCSUnderFileSystemTest { private static final String PATH = "path"; private static final String SRC = "src"; private static final String DST = "dst"; + private static final String KEY = "key"; private static final String BUCKET_NAME = "bucket"; @@ -52,13 +66,76 @@ public void before() { UnderFileSystemConfiguration.defaults(Configuration.global())); } + /** + * Test case for {@link GCSUnderFileSystem#getUnderFSType()}. + */ + @Test + public void getUnderFSType() { + Assert.assertEquals("gcs", mGCSUnderFileSystem.getUnderFSType()); + } + + /** + * Test case for {@link GCSUnderFileSystem#copyObject(String, String)}. + */ + @Test + public void testCopyObject() throws ServiceException { + // test successful copy object + when(mClient.copyObject(anyString(), anyString(), + anyString(), any(StorageObject.class), anyBoolean())).thenReturn(null); + boolean result = mGCSUnderFileSystem.copyObject(SRC, DST); + Assert.assertTrue(result); + + // test copy object exception + Mockito.when(mClient.copyObject(anyString(), anyString(), + anyString(), any(StorageObject.class), anyBoolean())).thenThrow(ServiceException.class); + try { + mGCSUnderFileSystem.copyObject(SRC, DST); + } catch (Exception e) { + Assert.assertTrue(e instanceof ServiceException); + } + } + + /** + * Test case for {@link GCSUnderFileSystem#createEmptyObject(String)}. + */ + @Test + public void testCreateEmptyObject() throws ServiceException { + + // test successful create empty object + Mockito.when(mClient.putObject(anyString(), + ArgumentMatchers.any(GSObject.class))).thenReturn(null); + boolean result = mGCSUnderFileSystem.createEmptyObject(KEY); + Assert.assertTrue(result); + + // test create empty object exception + Mockito.when(mClient.putObject(anyString(), + ArgumentMatchers.any(GSObject.class))).thenThrow(ServiceException.class); + try { + mGCSUnderFileSystem.createEmptyObject(KEY); + } catch (Exception e) { + Assert.assertTrue(e instanceof ServiceException); + } + } + + /** + * Test case for {@link GCSUnderFileSystem#createObject(String)}. + */ + @Test + public void testCreateObject() throws IOException, ServiceException { + // test successful create object + Mockito.when(mClient.putObject(anyString(), + ArgumentMatchers.any(GSObject.class))).thenReturn(null); + OutputStream result = mGCSUnderFileSystem.createObject(KEY); + Assert.assertTrue(result instanceof GCSOutputStream); + } + /** * Test case for {@link GCSUnderFileSystem#deleteDirectory(String, DeleteOptions)}. */ @Test public void deleteNonRecursiveOnServiceException() throws IOException, ServiceException { - when(mClient.listObjectsChunked(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), - ArgumentMatchers.anyString(), ArgumentMatchers.anyLong(), ArgumentMatchers.anyString())) + when(mClient.listObjectsChunked(anyString(), anyString(), + anyString(), ArgumentMatchers.anyLong(), anyString())) .thenThrow(ServiceException.class); boolean result = mGCSUnderFileSystem.deleteDirectory(PATH, @@ -71,8 +148,8 @@ public void deleteNonRecursiveOnServiceException() throws IOException, ServiceEx */ @Test public void deleteRecursiveOnServiceException() throws IOException, ServiceException { - when(mClient.listObjectsChunked(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), - ArgumentMatchers.anyString(), ArgumentMatchers.anyLong(), ArgumentMatchers.anyString())) + when(mClient.listObjectsChunked(anyString(), anyString(), + anyString(), ArgumentMatchers.anyLong(), anyString())) .thenThrow(ServiceException.class); boolean result = mGCSUnderFileSystem.deleteDirectory(PATH, @@ -85,11 +162,72 @@ public void deleteRecursiveOnServiceException() throws IOException, ServiceExcep */ @Test public void renameOnServiceException() throws IOException, ServiceException { - when(mClient.listObjectsChunked(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), - ArgumentMatchers.anyString(), ArgumentMatchers.anyLong(), ArgumentMatchers.anyString())) + when(mClient.listObjectsChunked(anyString(), anyString(), + anyString(), ArgumentMatchers.anyLong(), anyString())) .thenThrow(ServiceException.class); boolean result = mGCSUnderFileSystem.renameFile(SRC, DST); assertFalse(result); } + /** + * Test case for {@link GCSUnderFileSystem#getFolderSuffix()}. + */ + @Test + public void testGetFolderSuffix() { + Assert.assertEquals("/", mGCSUnderFileSystem.getFolderSuffix()); + } + + /** + * Test case for {@link GCSUnderFileSystem#getObjectListingChunk(String, String, String)}. + */ + @Test + public void testGetObjectListingChunk() throws ServiceException { + // test successful get object listing chunk + Mockito.when(mClient.listObjectsChunked(anyString(), anyString(), + anyString(), anyLong(), anyString())).thenReturn(new StorageObjectsChunk( + null, null, null, null, null)); + String delimiter = "/"; + String prefix = ""; + Object result = mGCSUnderFileSystem.getObjectListingChunk(KEY, delimiter, prefix); + Assert.assertTrue(result instanceof StorageObjectsChunk); + } + + /** + * Test case for {@link GCSUnderFileSystem#isDirectory(String)}. + */ + @Test + public void testIsDirectory() throws IOException { + Assert.assertTrue(mGCSUnderFileSystem.isDirectory("/")); + } + + /** + * Test case for {@link GCSUnderFileSystem#openPositionRead(String, long)}. + */ + @Test + public void testOpenPositionRead() { + PositionReader result = mGCSUnderFileSystem.openPositionRead(KEY, 1L); + Assert.assertTrue(result instanceof GCSPositionReader); + } + + /** + * Test case for {@link GCSUnderFileSystem#getRootKey()}. + */ + @Test + public void testGetRootKey() { + Assert.assertEquals(Constants.HEADER_GCS + BUCKET_NAME, mGCSUnderFileSystem.getRootKey()); + } + + /** + * Test case for {@link GCSUnderFileSystem#openObject(String, OpenOptions, RetryPolicy)}. + */ + @Test + public void testOpenObject() throws IOException, ServiceException { + // test successful open object + Mockito.when(mClient.getObject(anyString(), anyString())) + .thenReturn(new GSObject()); + OpenOptions options = OpenOptions.defaults(); + RetryPolicy retryPolicy = new CountingRetry(1); + InputStream result = mGCSUnderFileSystem.openObject(KEY, options, retryPolicy); + Assert.assertTrue(result instanceof GCSInputStream); + } } diff --git a/dora/underfs/obs/src/main/java/alluxio/underfs/obs/OBSUnderFileSystem.java b/dora/underfs/obs/src/main/java/alluxio/underfs/obs/OBSUnderFileSystem.java index 0ee9f4c6f2f7..18fcd1ec97d7 100644 --- a/dora/underfs/obs/src/main/java/alluxio/underfs/obs/OBSUnderFileSystem.java +++ b/dora/underfs/obs/src/main/java/alluxio/underfs/obs/OBSUnderFileSystem.java @@ -286,7 +286,7 @@ protected ObjectListingChunk getObjectListingChunk(String key, boolean recursive } // Get next chunk of listing result - private ObjectListing getObjectListingChunk(ListObjectsRequest request) { + protected ObjectListing getObjectListingChunk(ListObjectsRequest request) { ObjectListing result; try { result = mClient.listObjects(request); diff --git a/dora/underfs/obs/src/main/java/alluxio/underfs/obs/OBSUnderFileSystemFactory.java b/dora/underfs/obs/src/main/java/alluxio/underfs/obs/OBSUnderFileSystemFactory.java index 35ebaa559a07..98f8ef3d614e 100644 --- a/dora/underfs/obs/src/main/java/alluxio/underfs/obs/OBSUnderFileSystemFactory.java +++ b/dora/underfs/obs/src/main/java/alluxio/underfs/obs/OBSUnderFileSystemFactory.java @@ -38,7 +38,8 @@ public OBSUnderFileSystemFactory() { @Override public UnderFileSystem create(String path, UnderFileSystemConfiguration conf) { - Preconditions.checkNotNull(path, "path"); + Preconditions.checkNotNull(path, "Unable to create UnderFileSystem instance:" + + " URI path should not be null"); if (checkOBSCredentials(conf)) { try { diff --git a/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSInputStreamTest.java b/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSInputStreamTest.java index 6ca9b340d21a..093c1219c712 100644 --- a/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSInputStreamTest.java +++ b/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSInputStreamTest.java @@ -12,8 +12,6 @@ package alluxio.underfs.obs; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -25,8 +23,10 @@ import alluxio.retry.CountingRetry; import com.obs.services.ObsClient; +import com.obs.services.exception.ObsException; import com.obs.services.model.GetObjectRequest; import com.obs.services.model.ObsObject; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -51,6 +51,8 @@ public class OBSInputStreamTest { private InputStream[] mInputStreamSpy; private ObsObject[] mObjects; + private static final byte[] INPUT_ARRAY = new byte[] {0, 1, 2}; + /** * The exception expected to be thrown. */ @@ -61,19 +63,18 @@ public class OBSInputStreamTest { public void setUp() throws IOException { mObsClient = mock(ObsClient.class); - byte[] input = new byte[]{1, 2, 3}; - mObjects = new ObsObject[input.length]; - mInputStreamSpy = new InputStream[input.length]; - for (int i = 0; i < input.length; ++i) { + mObjects = new ObsObject[INPUT_ARRAY.length]; + mInputStreamSpy = new InputStream[INPUT_ARRAY.length]; + for (int i = 0; i < INPUT_ARRAY.length; ++i) { final long pos = i; mObjects[i] = mock(ObsObject.class); when(mObsClient.getObject(argThat(argument -> { if (argument instanceof GetObjectRequest) { - return ((GetObjectRequest) argument).getRangeStart() == pos; + return argument.getRangeStart() == pos; } return false; }))).thenReturn(mObjects[i]); - byte[] mockInput = Arrays.copyOfRange(input, i, input.length); + byte[] mockInput = Arrays.copyOfRange(INPUT_ARRAY, i, INPUT_ARRAY.length); mInputStreamSpy[i] = spy(new ByteArrayInputStream(mockInput)); when(mObjects[i].getObjectContent()).thenReturn(mInputStreamSpy[i]); } @@ -92,23 +93,36 @@ public void close() throws IOException { @Test public void readInt() throws IOException { - assertEquals(1, mOBSInputStream.read()); - assertEquals(2, mOBSInputStream.read()); - assertEquals(3, mOBSInputStream.read()); + for (int i = 0; i < INPUT_ARRAY.length; ++i) { + Assert.assertEquals(INPUT_ARRAY[i], mOBSInputStream.read()); + } } @Test public void readByteArray() throws IOException { - byte[] bytes = new byte[3]; - int readCount = mOBSInputStream.read(bytes, 0, 3); - assertEquals(3, readCount); - assertArrayEquals(new byte[]{1, 2, 3}, bytes); + byte[] bytes = new byte[INPUT_ARRAY.length]; + int readCount = mOBSInputStream.read(bytes, 0, INPUT_ARRAY.length); + Assert.assertEquals(INPUT_ARRAY.length, readCount); + Assert.assertArrayEquals(INPUT_ARRAY, bytes); } @Test public void skip() throws IOException { - assertEquals(1, mOBSInputStream.read()); + Assert.assertEquals(0, mOBSInputStream.read()); mOBSInputStream.skip(1); - assertEquals(3, mOBSInputStream.read()); + Assert.assertEquals(2, mOBSInputStream.read()); + } + + @Test + public void testException() { + + for (int i = 0; i < INPUT_ARRAY.length; ++i) { + when(mObjects[i].getObjectContent()).thenThrow(ObsException.class); + try { + mOBSInputStream.read(); + } catch (Exception e) { + Assert.assertTrue(e instanceof IOException); + } + } } } diff --git a/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSPositionReaderTest.java b/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSPositionReaderTest.java new file mode 100644 index 000000000000..567ef915d7d3 --- /dev/null +++ b/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSPositionReaderTest.java @@ -0,0 +1,84 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.underfs.obs; + +import com.obs.services.ObsClient; +import com.obs.services.exception.ObsException; +import com.obs.services.model.GetObjectRequest; +import com.obs.services.model.ObsObject; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; + +import java.io.IOException; +import java.io.InputStream; + +public class OBSPositionReaderTest { + /** + * The OBS Position Reader. + */ + private OBSPositionReader mOBSPositionReader; + /** + * The OBS Client. + */ + private ObsClient mClient; + /** + * The Bucket Name. + */ + private final String mBucketName = "bucket"; + /** + * The Path (or the Key). + */ + private final String mPath = "path"; + /** + * The File Length. + */ + private final long mFileLength = 100L; + + @Before + public void before() throws Exception { + mClient = Mockito.mock(ObsClient.class); + mOBSPositionReader = new OBSPositionReader(mClient, mBucketName, mPath, mFileLength); + } + + /** + * Test case for {@link OBSPositionReader#openObjectInputStream(long, int)}. + */ + @Test + public void openObjectInputStream() throws Exception { + ObsObject object = Mockito.mock(ObsObject.class); + InputStream inputStream = Mockito.mock(InputStream.class); + Mockito.when(mClient.getObject(ArgumentMatchers.any( + GetObjectRequest.class))).thenReturn(object); + Mockito.when(object.getObjectContent()).thenReturn(inputStream); + + // test successful open object input stream + long position = 0L; + int bytesToRead = 10; + Object objectInputStream = mOBSPositionReader.openObjectInputStream(position, bytesToRead); + Assert.assertTrue(objectInputStream instanceof InputStream); + + // test open object input stream with exception + Mockito.when(mClient.getObject(ArgumentMatchers.any(GetObjectRequest.class))) + .thenThrow(ObsException.class); + try { + mOBSPositionReader.openObjectInputStream(position, bytesToRead); + } catch (Exception e) { + Assert.assertTrue(e instanceof IOException); + String errorMessage = String + .format("Failed to get object: %s bucket: %s", mPath, mBucketName); + Assert.assertEquals(errorMessage, e.getMessage()); + } + } +} diff --git a/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemFactoryTest.java b/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemFactoryTest.java index e973df8c83dc..2f18ac10c38b 100644 --- a/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemFactoryTest.java +++ b/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemFactoryTest.java @@ -11,13 +11,20 @@ package alluxio.underfs.obs; +import alluxio.conf.AlluxioConfiguration; import alluxio.conf.Configuration; +import alluxio.conf.PropertyKey; +import alluxio.underfs.UnderFileSystem; +import alluxio.underfs.UnderFileSystemConfiguration; import alluxio.underfs.UnderFileSystemFactory; import alluxio.underfs.UnderFileSystemFactoryRegistry; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertFalse; + /** * Unit tests for the {@link OBSUnderFileSystemFactory}. */ @@ -26,12 +33,84 @@ public class OBSUnderFileSystemFactoryTest { /** * Tests that the OBS UFS module correctly accepts paths that begin with obs://. */ + + private final String mObsPath = "obs://test-bucket/path"; + private final String mObsAccessKey = "test-access-key"; + private final String mObsSecretKey = "test-secret-key"; + private final String mObsEndpoint = "test-endpoint"; + private AlluxioConfiguration mAlluxioConf; + private UnderFileSystemConfiguration mConf; + private UnderFileSystemFactory mFactory; + + @Before + public void setUp() { + Configuration.set(PropertyKey.OBS_ACCESS_KEY, mObsAccessKey); + Configuration.set(PropertyKey.OBS_SECRET_KEY, mObsSecretKey); + Configuration.set(PropertyKey.OBS_ENDPOINT, mObsEndpoint); + mAlluxioConf = Configuration.global(); + mConf = UnderFileSystemConfiguration.defaults(mAlluxioConf); + mFactory = UnderFileSystemFactoryRegistry.find(mObsPath, mConf); + } + @Test public void factory() { - UnderFileSystemFactory factory = UnderFileSystemFactoryRegistry.find("obs://bucket/key", - Configuration.global()); - Assert.assertNotNull( - "A UnderFileSystemFactory should exist for obs paths when using this module", factory); + "A UnderFileSystemFactory should exist for obs paths when using this module", mFactory); + } + + /** + * Test case for {@link OBSUnderFileSystemFactory#create(String, UnderFileSystemConfiguration)}. + */ + @Test + public void createInstanceWithNullPath() { + Exception e = Assert.assertThrows(NullPointerException.class, () -> mFactory.create( + null, mConf)); + Assert.assertTrue(e.getMessage().contains("Unable to create UnderFileSystem instance: URI " + + "path should not be null")); + } + + /** + * Test case for {@link OBSUnderFileSystemFactory#create(String, UnderFileSystemConfiguration)}. + */ + @Test + public void createInstanceWithPath() { + UnderFileSystem ufs = mFactory.create(mObsPath, mConf); + Assert.assertNotNull(ufs); + Assert.assertTrue(ufs instanceof OBSUnderFileSystem); + } + + /** + * Test case for {@link OBSUnderFileSystemFactory#create(String, UnderFileSystemConfiguration)}. + */ + @Test + public void createInstanceWithoutCredentials() { + Configuration.unset(PropertyKey.OBS_ACCESS_KEY); + Configuration.unset(PropertyKey.OBS_SECRET_KEY); + Configuration.unset(PropertyKey.OBS_ENDPOINT); + mAlluxioConf = Configuration.global(); + mConf = UnderFileSystemConfiguration.defaults(mAlluxioConf); + + try { + mFactory.create(mObsPath, mConf); + } catch (Exception e) { + Assert.assertTrue(e instanceof RuntimeException); + Assert.assertTrue(e.getMessage().contains("OBS credentials or endpoint not available, " + + "cannot create OBS Under File System.")); + } + Exception e = Assert.assertThrows(RuntimeException.class, () -> mFactory.create( + mObsPath, mConf)); + Assert.assertTrue(e.getMessage().contains("OBS credentials or endpoint not available, " + + "cannot create OBS Under File System.")); + } + + /** + * Test case for {@link OBSUnderFileSystemFactory#supportsPath(String)}. + */ + @Test + public void supportsPath() { + Assert.assertTrue(mFactory.supportsPath(mObsPath)); + assertFalse(mFactory.supportsPath(null)); + assertFalse(mFactory.supportsPath("Invalid_Path")); + assertFalse(mFactory.supportsPath("hdfs://test-bucket/path")); } } diff --git a/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemTest.java b/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemTest.java index cc65883a090f..657efa081642 100644 --- a/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemTest.java +++ b/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemTest.java @@ -12,14 +12,18 @@ package alluxio.underfs.obs; import alluxio.AlluxioURI; +import alluxio.Constants; +import alluxio.PositionReader; import alluxio.conf.Configuration; +import alluxio.retry.CountingRetry; +import alluxio.retry.RetryPolicy; import alluxio.underfs.UnderFileSystemConfiguration; import alluxio.underfs.options.DeleteOptions; +import alluxio.underfs.options.OpenOptions; import com.obs.services.ObsClient; import com.obs.services.exception.ObsException; -import com.obs.services.model.ListObjectsRequest; -import com.obs.services.model.ObjectMetadata; +import com.obs.services.model.*; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -27,7 +31,12 @@ import org.mockito.Mockito; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collections; import java.util.Date; +import java.util.List; /** * Unit tests for the {@link OBSUnderFileSystem}. @@ -40,6 +49,7 @@ public class OBSUnderFileSystemTest { private static final String PATH = "path"; private static final String SRC = "src"; private static final String DST = "dst"; + private static final String KEY = "key"; private static final String BUCKET_NAME = "bucket"; private static final String BUCKET_TYPE = "obs"; @@ -48,12 +58,110 @@ public class OBSUnderFileSystemTest { * Set up. */ @Before - public void before() throws InterruptedException, ObsException { + public void before() throws ObsException { mClient = Mockito.mock(ObsClient.class); mOBSUnderFileSystem = new OBSUnderFileSystem(new AlluxioURI(""), mClient, BUCKET_NAME, BUCKET_TYPE, UnderFileSystemConfiguration.defaults(Configuration.global())); } + /** + * Test case for {@link OBSUnderFileSystem#getUnderFSType()}. + */ + @Test + public void getUnderFSType() { + Assert.assertEquals("obs", mOBSUnderFileSystem.getUnderFSType()); + } + + /** + * Test case for {@link OBSUnderFileSystem#copyObject(String, String)}. + */ + @Test + public void testCopyObject() { + // test successful copy object + Mockito.when(mClient.copyObject(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyString())).thenReturn(null); + boolean result = mOBSUnderFileSystem.copyObject(SRC, DST); + Assert.assertTrue(result); + + // test copy object exception + Mockito.when(mClient.copyObject(ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), + ArgumentMatchers.anyString())).thenThrow(ObsException.class); + try { + mOBSUnderFileSystem.copyObject(SRC, DST); + } catch (Exception e) { + Assert.assertTrue(e instanceof ObsException); + } + } + + /** + * Test case for {@link OBSUnderFileSystem#createEmptyObject(String)}. + */ + @Test + public void testCreateEmptyObject() { + // test successful create empty object + Mockito.when(mClient.putObject(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), + ArgumentMatchers.any(InputStream.class), ArgumentMatchers.any(ObjectMetadata.class))) + .thenReturn(null); + boolean result = mOBSUnderFileSystem.createEmptyObject(KEY); + Assert.assertTrue(result); + + // test create empty object exception + Mockito.when(mClient.putObject(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), + ArgumentMatchers.any(InputStream.class), ArgumentMatchers.any(ObjectMetadata.class))) + .thenThrow(ObsException.class); + try { + mOBSUnderFileSystem.createEmptyObject(KEY); + } catch (Exception e) { + Assert.assertTrue(e instanceof ObsException); + } + } + + /** + * Test case for {@link OBSUnderFileSystem#createObject(String)}. + */ + @Test + public void testCreateObject() throws IOException { + // test successful create object + Mockito.when(mClient.putObject(ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), ArgumentMatchers.any(InputStream.class), + ArgumentMatchers.any(ObjectMetadata.class))).thenReturn(null); + OutputStream result = mOBSUnderFileSystem.createObject(KEY); + Assert.assertTrue(result instanceof OBSOutputStream); + } + + /** + * Test case for {@link OBSUnderFileSystem#deleteObjects(List)}. + */ + @Test + public void testDeleteObjects() throws IOException { + String[] stringKeys = new String[]{"key1", "key2", "key3"}; + List keys = new ArrayList<>(); + Collections.addAll(keys, stringKeys); + List deletedObjects = + new ArrayList<>(); + for (String key : keys) { + DeleteObjectsResult.DeleteObjectResult deletedObject = + new DeleteObjectsResult.DeleteObjectResult(key, "", true, ""); + deletedObjects.add(deletedObject); + } + // test successful delete objects + Mockito.when(mClient.deleteObjects(ArgumentMatchers.any(DeleteObjectsRequest.class))) + .thenReturn(new DeleteObjectsResult(deletedObjects, null)); + + List result = mOBSUnderFileSystem.deleteObjects(keys); + Assert.assertEquals(keys, result); + + // test delete objects exception + Mockito.when(mClient.deleteObjects(ArgumentMatchers.any(DeleteObjectsRequest.class))) + .thenThrow(ObsException.class); + try { + mOBSUnderFileSystem.deleteObjects(keys); + } catch (Exception e) { + Assert.assertTrue(e instanceof IOException); + } + } + /** * Test case for {@link OBSUnderFileSystem#deleteDirectory(String, DeleteOptions)}. */ @@ -163,4 +271,65 @@ public void nullObjectMetaTest() throws Exception { Assert.assertNotNull(mOBSUnderFileSystem.getObjectStatus("pfs_file1")); Assert.assertNotNull(mOBSUnderFileSystem.getObjectStatus("dir1")); } + + /** + * Test case for {@link OBSUnderFileSystem#getFolderSuffix()}. + */ + @Test + public void testGetFolderSuffix() { + Assert.assertEquals("/", mOBSUnderFileSystem.getFolderSuffix()); + } + + /** + * Test case for {@link OBSUnderFileSystem#getObjectListingChunk(ListObjectsRequest)}. + */ + @Test + public void testGetObjectListingChunk() { + // test successful get object listing chunk + Mockito.when(mClient.listObjects(ArgumentMatchers.any(ListObjectsRequest.class))) + .thenReturn(new ObjectListing(null, null, null, false, + null, null, 0, null, null, null)); + ListObjectsRequest request = new ListObjectsRequest(); + HeaderResponse result = mOBSUnderFileSystem.getObjectListingChunk(request); + Assert.assertTrue(result instanceof ObjectListing); + } + + /** + * Test case for {@link OBSUnderFileSystem#isDirectory(String)}. + */ + @Test + public void testIsDirectory() throws IOException { + Assert.assertTrue(mOBSUnderFileSystem.isDirectory("/")); + } + + /** + * Test case for {@link OBSUnderFileSystem#openPositionRead(String, long)}. + */ + @Test + public void testOpenPositionRead() { + PositionReader result = mOBSUnderFileSystem.openPositionRead(KEY, 1L); + Assert.assertTrue(result instanceof OBSPositionReader); + } + + /** + * Test case for {@link OBSUnderFileSystem#getRootKey()}. + */ + @Test + public void testGetRootKey() { + Assert.assertEquals(Constants.HEADER_OBS + BUCKET_NAME, mOBSUnderFileSystem.getRootKey()); + } + + /** + * Test case for {@link OBSUnderFileSystem#openObject(String, OpenOptions, RetryPolicy)}. + */ + @Test + public void testOpenObject() throws IOException { + // test successful open object + Mockito.when(mClient.getObject(ArgumentMatchers.anyString(), ArgumentMatchers.anyString())) + .thenReturn(new ObsObject()); + OpenOptions options = OpenOptions.defaults(); + RetryPolicy retryPolicy = new CountingRetry(1); + InputStream result = mOBSUnderFileSystem.openObject(KEY, options, retryPolicy); + Assert.assertTrue(result instanceof OBSInputStream); + } } diff --git a/dora/underfs/oss/src/main/java/alluxio/underfs/oss/OSSUnderFileSystem.java b/dora/underfs/oss/src/main/java/alluxio/underfs/oss/OSSUnderFileSystem.java index 393d6f2ee7e6..fecd20a36271 100644 --- a/dora/underfs/oss/src/main/java/alluxio/underfs/oss/OSSUnderFileSystem.java +++ b/dora/underfs/oss/src/main/java/alluxio/underfs/oss/OSSUnderFileSystem.java @@ -280,7 +280,7 @@ protected ObjectListingChunk getObjectListingChunk(String key, boolean recursive } // Get next chunk of listing result - private ObjectListing getObjectListingChunk(ListObjectsRequest request) { + protected ObjectListing getObjectListingChunk(ListObjectsRequest request) { ObjectListing result; try { result = mClient.listObjects(request); diff --git a/dora/underfs/oss/src/main/java/alluxio/underfs/oss/OSSUnderFileSystemFactory.java b/dora/underfs/oss/src/main/java/alluxio/underfs/oss/OSSUnderFileSystemFactory.java index 07ee2098502e..6982783e0158 100644 --- a/dora/underfs/oss/src/main/java/alluxio/underfs/oss/OSSUnderFileSystemFactory.java +++ b/dora/underfs/oss/src/main/java/alluxio/underfs/oss/OSSUnderFileSystemFactory.java @@ -37,7 +37,8 @@ public OSSUnderFileSystemFactory() {} @Override public UnderFileSystem create(String path, UnderFileSystemConfiguration conf) { - Preconditions.checkNotNull(path, "path"); + Preconditions.checkNotNull(path, "Unable to create UnderFileSystem instance:" + + " URI path should not be null"); if (checkOSSCredentials(conf)) { try { diff --git a/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSInputStreamTest.java b/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSInputStreamTest.java index 0e8d1584e0f4..f2146dec1525 100644 --- a/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSInputStreamTest.java +++ b/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSInputStreamTest.java @@ -25,7 +25,9 @@ import alluxio.retry.CountingRetry; import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; import com.aliyun.oss.model.OSSObject; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -49,6 +51,7 @@ public class OSSInputStreamTest { private OSS mOssClient; private InputStream[] mInputStreamSpy; private OSSObject[] mOssObject; + private static final byte[] INPUT_ARRAY = new byte[] {0, 1, 2}; /** * The exception expected to be thrown. @@ -60,10 +63,9 @@ public class OSSInputStreamTest { public void setUp() throws IOException { mOssClient = mock(OSS.class); - byte[] input = new byte[] {1, 2, 3}; - mOssObject = new OSSObject[input.length]; - mInputStreamSpy = new InputStream[input.length]; - for (int i = 0; i < input.length; ++i) { + mOssObject = new OSSObject[INPUT_ARRAY.length]; + mInputStreamSpy = new InputStream[INPUT_ARRAY.length]; + for (int i = 0; i < INPUT_ARRAY.length; ++i) { final long pos = (long) i; mOssObject[i] = mock(OSSObject.class); when(mOssClient.getObject(argThat(argument -> { @@ -73,7 +75,7 @@ public void setUp() throws IOException { return false; }))) .thenReturn(mOssObject[i]); - byte[] mockInput = Arrays.copyOfRange(input, i, input.length); + byte[] mockInput = Arrays.copyOfRange(INPUT_ARRAY, i, INPUT_ARRAY.length); mInputStreamSpy[i] = spy(new ByteArrayInputStream(mockInput)); when(mOssObject[i].getObjectContent()).thenReturn(mInputStreamSpy[i]); } @@ -92,23 +94,37 @@ public void close() throws IOException { @Test public void readInt() throws IOException { - assertEquals(1, mOssInputStream.read()); - assertEquals(2, mOssInputStream.read()); - assertEquals(3, mOssInputStream.read()); + for (int i = 0; i < INPUT_ARRAY.length; ++i) { + Assert.assertEquals(INPUT_ARRAY[i], mOssInputStream.read()); + } } @Test public void readByteArray() throws IOException { - byte[] bytes = new byte[3]; - int readCount = mOssInputStream.read(bytes, 0, 3); - assertEquals(3, readCount); - assertArrayEquals(new byte[] {1, 2, 3}, bytes); + byte[] bytes = new byte[INPUT_ARRAY.length]; + int readCount = mOssInputStream.read(bytes, 0, INPUT_ARRAY.length); + assertEquals(INPUT_ARRAY.length, readCount); + assertArrayEquals(INPUT_ARRAY, bytes); } @Test public void skip() throws IOException { - assertEquals(1, mOssInputStream.read()); + assertEquals(0, mOssInputStream.read()); mOssInputStream.skip(1); - assertEquals(3, mOssInputStream.read()); + assertEquals(2, mOssInputStream.read()); + } + + @Test + public void testException() { + + for (int i = 0; i < INPUT_ARRAY.length; ++i) { + when(mOssObject[i].getObjectContent()).thenThrow( + new OSSException("", String.valueOf(404), "", "", "", "", null)); + try { + mOssInputStream.read(); + } catch (Exception e) { + Assert.assertTrue(e instanceof IOException); + } + } } } diff --git a/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSPositionReaderTest.java b/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSPositionReaderTest.java new file mode 100644 index 000000000000..97a10c0b34d1 --- /dev/null +++ b/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSPositionReaderTest.java @@ -0,0 +1,84 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.underfs.oss; + +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.GetObjectRequest; +import com.aliyun.oss.model.OSSObject; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; + +import java.io.IOException; +import java.io.InputStream; + +public class OSSPositionReaderTest { + /** + * The OSS Position Reader. + */ + private OSSPositionReader mOSSPositionReader; + /** + * The OSS Client. + */ + private OSSClient mClient; + /** + * The Bucket Name. + */ + private final String mBucketName = "bucket"; + /** + * The Path (or the Key). + */ + private final String mPath = "path"; + /** + * The File Length. + */ + private final long mFileLength = 100L; + + @Before + public void before() throws Exception { + mClient = Mockito.mock(OSSClient.class); + mOSSPositionReader = new OSSPositionReader(mClient, mBucketName, mPath, mFileLength); + } + + /** + * Test case for {@link OSSPositionReader#openObjectInputStream(long, int)}. + */ + @Test + public void openObjectInputStream() throws Exception { + OSSObject object = Mockito.mock(OSSObject.class); + InputStream inputStream = Mockito.mock(InputStream.class); + Mockito.when(mClient.getObject(ArgumentMatchers.any( + GetObjectRequest.class))).thenReturn(object); + Mockito.when(object.getObjectContent()).thenReturn(inputStream); + + // test successful open object input stream + long position = 0L; + int bytesToRead = 10; + Object objectInputStream = mOSSPositionReader.openObjectInputStream(position, bytesToRead); + Assert.assertTrue(objectInputStream instanceof InputStream); + + // test open object input stream with exception + Mockito.when(mClient.getObject(ArgumentMatchers.any(GetObjectRequest.class))) + .thenThrow(OSSException.class); + try { + mOSSPositionReader.openObjectInputStream(position, bytesToRead); + } catch (Exception e) { + Assert.assertTrue(e instanceof IOException); + String errorMessage = String + .format("Failed to get object: %s bucket: %s", mPath, mBucketName); + Assert.assertEquals(errorMessage, e.getMessage()); + } + } +} diff --git a/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemFactoryTest.java b/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemFactoryTest.java index 19a7dcdb03c6..e900b8d69a7a 100644 --- a/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemFactoryTest.java +++ b/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemFactoryTest.java @@ -11,13 +11,20 @@ package alluxio.underfs.oss; +import alluxio.conf.AlluxioConfiguration; import alluxio.conf.Configuration; +import alluxio.conf.PropertyKey; +import alluxio.underfs.UnderFileSystem; +import alluxio.underfs.UnderFileSystemConfiguration; import alluxio.underfs.UnderFileSystemFactory; import alluxio.underfs.UnderFileSystemFactoryRegistry; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertFalse; + /** * Unit tests for the {@link OSSUnderFileSystemFactory}. */ @@ -26,12 +33,84 @@ public class OSSUnderFileSystemFactoryTest { /** * Tests that the OSS UFS module correctly accepts paths that begin with oss://. */ + + private final String mOssPath = "oss://test-bucket/path"; + private final String mOssAccessKey = "test-access-key"; + private final String mOssSecretKey = "test-secret-key"; + private final String mOssEndpoint = "test-endpoint"; + private AlluxioConfiguration mAlluxioConf; + private UnderFileSystemConfiguration mConf; + private UnderFileSystemFactory mFactory; + + @Before + public void setUp() { + Configuration.set(PropertyKey.OSS_ACCESS_KEY, mOssAccessKey); + Configuration.set(PropertyKey.OSS_SECRET_KEY, mOssSecretKey); + Configuration.set(PropertyKey.OSS_ENDPOINT_KEY, mOssEndpoint); + mAlluxioConf = Configuration.global(); + mConf = UnderFileSystemConfiguration.defaults(mAlluxioConf); + mFactory = UnderFileSystemFactoryRegistry.find(mOssPath, mConf); + } + @Test public void factory() { - UnderFileSystemFactory factory = UnderFileSystemFactoryRegistry.find("oss://test-bucket/path", - Configuration.global()); - Assert.assertNotNull( - "A UnderFileSystemFactory should exist for oss paths when using this module", factory); + "A UnderFileSystemFactory should exist for oss paths when using this module", mFactory); + } + + /** + * Test case for {@link OSSUnderFileSystemFactory#create(String, UnderFileSystemConfiguration)}. + */ + @Test + public void createInstanceWithNullPath() { + Exception e = Assert.assertThrows(NullPointerException.class, () -> mFactory.create( + null, mConf)); + Assert.assertTrue(e.getMessage().contains("Unable to create UnderFileSystem instance: URI " + + "path should not be null")); + } + + /** + * Test case for {@link OSSUnderFileSystemFactory#create(String, UnderFileSystemConfiguration)}. + */ + @Test + public void createInstanceWithPath() { + UnderFileSystem ufs = mFactory.create(mOssPath, mConf); + Assert.assertNotNull(ufs); + Assert.assertTrue(ufs instanceof OSSUnderFileSystem); + } + + /** + * Test case for {@link OSSUnderFileSystemFactory#create(String, UnderFileSystemConfiguration)}. + */ + @Test + public void createInstanceWithoutCredentials() { + Configuration.unset(PropertyKey.OSS_ACCESS_KEY); + Configuration.unset(PropertyKey.OSS_SECRET_KEY); + Configuration.unset(PropertyKey.OSS_ENDPOINT_KEY); + mAlluxioConf = Configuration.global(); + mConf = UnderFileSystemConfiguration.defaults(mAlluxioConf); + + try { + mFactory.create(mOssPath, mConf); + } catch (Exception e) { + Assert.assertTrue(e instanceof RuntimeException); + Assert.assertTrue(e.getMessage().contains("OSS Credentials not available, " + + "cannot create OSS Under File System.")); + } + Exception e = Assert.assertThrows(RuntimeException.class, () -> mFactory.create( + mOssPath, mConf)); + Assert.assertTrue(e.getMessage().contains("OSS Credentials not available, " + + "cannot create OSS Under File System.")); + } + + /** + * Test case for {@link OSSUnderFileSystemFactory#supportsPath(String)}. + */ + @Test + public void supportsPath() { + Assert.assertTrue(mFactory.supportsPath(mOssPath)); + assertFalse(mFactory.supportsPath(null)); + assertFalse(mFactory.supportsPath("Invalid_Path")); + assertFalse(mFactory.supportsPath("hdfs://test-bucket/path")); } } diff --git a/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemTest.java b/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemTest.java index bf13ed33b4f1..11a4011d0823 100644 --- a/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemTest.java +++ b/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemTest.java @@ -12,13 +12,24 @@ package alluxio.underfs.oss; import alluxio.AlluxioURI; +import alluxio.Constants; +import alluxio.PositionReader; import alluxio.conf.Configuration; +import alluxio.retry.CountingRetry; +import alluxio.retry.RetryPolicy; import alluxio.underfs.UnderFileSystemConfiguration; import alluxio.underfs.options.DeleteOptions; +import alluxio.underfs.options.OpenOptions; import com.aliyun.oss.OSSClient; import com.aliyun.oss.ServiceException; +import com.aliyun.oss.model.DeleteObjectsRequest; +import com.aliyun.oss.model.DeleteObjectsResult; +import com.aliyun.oss.model.GenericResult; import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.OSSObject; +import com.aliyun.oss.model.ObjectListing; +import com.aliyun.oss.model.ObjectMetadata; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -26,6 +37,11 @@ import org.mockito.Mockito; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; /** * Unit tests for the {@link OSSUnderFileSystem}. @@ -38,6 +54,7 @@ public class OSSUnderFileSystemTest { private static final String PATH = "path"; private static final String SRC = "src"; private static final String DST = "dst"; + private static final String KEY = "key"; private static final String BUCKET_NAME = "bucket"; @@ -45,13 +62,104 @@ public class OSSUnderFileSystemTest { * Set up. */ @Before - public void before() throws InterruptedException, ServiceException { + public void before() throws ServiceException { mClient = Mockito.mock(OSSClient.class); mOSSUnderFileSystem = new OSSUnderFileSystem(new AlluxioURI(""), mClient, BUCKET_NAME, UnderFileSystemConfiguration.defaults(Configuration.global())); } + /** + * Test case for {@link OSSUnderFileSystem#getUnderFSType()}. + */ + @Test + public void getUnderFSType() { + Assert.assertEquals("oss", mOSSUnderFileSystem.getUnderFSType()); + } + + /** + * Test case for {@link OSSUnderFileSystem#copyObject(String, String)}. + */ + @Test + public void testCopyObject() { + // test successful copy object + Mockito.when(mClient.copyObject(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyString())).thenReturn(null); + boolean result = mOSSUnderFileSystem.copyObject(SRC, DST); + Assert.assertTrue(result); + + // test copy object exception + Mockito.when(mClient.copyObject(ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), + ArgumentMatchers.anyString())).thenThrow(ServiceException.class); + try { + mOSSUnderFileSystem.copyObject(SRC, DST); + } catch (Exception e) { + Assert.assertTrue(e instanceof ServiceException); + } + } + + /** + * Test case for {@link OSSUnderFileSystem#createEmptyObject(String)}. + */ + @Test + public void testCreateEmptyObject() { + // test successful create empty object + Mockito.when(mClient.putObject(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), + ArgumentMatchers.any(InputStream.class), ArgumentMatchers.any(ObjectMetadata.class))) + .thenReturn(null); + boolean result = mOSSUnderFileSystem.createEmptyObject(KEY); + Assert.assertTrue(result); + + // test create empty object exception + Mockito.when(mClient.putObject(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), + ArgumentMatchers.any(InputStream.class), ArgumentMatchers.any(ObjectMetadata.class))) + .thenThrow(ServiceException.class); + try { + mOSSUnderFileSystem.createEmptyObject(KEY); + } catch (Exception e) { + Assert.assertTrue(e instanceof ServiceException); + } + } + + /** + * Test case for {@link OSSUnderFileSystem#createObject(String)}. + */ + @Test + public void testCreateObject() throws IOException { + // test successful create object + Mockito.when(mClient.putObject(ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), ArgumentMatchers.any(InputStream.class), + ArgumentMatchers.any(ObjectMetadata.class))).thenReturn(null); + OutputStream result = mOSSUnderFileSystem.createObject(KEY); + Assert.assertTrue(result instanceof OSSOutputStream); + } + + /** + * Test case for {@link OSSUnderFileSystem#deleteObjects(List)}. + */ + @Test + public void testDeleteObjects() throws IOException { + String[] stringKeys = new String[]{"key1", "key2", "key3"}; + List keys = new ArrayList<>(); + Collections.addAll(keys, stringKeys); + // test successful delete objects + Mockito.when(mClient.deleteObjects(ArgumentMatchers.any(DeleteObjectsRequest.class))) + .thenReturn(new DeleteObjectsResult(keys)); + + List result = mOSSUnderFileSystem.deleteObjects(keys); + Assert.assertEquals(keys, result); + + // test delete objects exception + Mockito.when(mClient.deleteObjects(ArgumentMatchers.any(DeleteObjectsRequest.class))) + .thenThrow(ServiceException.class); + try { + mOSSUnderFileSystem.deleteObjects(keys); + } catch (Exception e) { + Assert.assertTrue(e instanceof IOException); + } + } + /** * Test case for {@link OSSUnderFileSystem#deleteDirectory(String, DeleteOptions)}. */ @@ -89,4 +197,77 @@ public void renameOnServiceException() throws IOException { boolean result = mOSSUnderFileSystem.renameFile(SRC, DST); Assert.assertFalse(result); } + + /** + * Test case for {@link OSSUnderFileSystem#renameFile(String, String)}. + */ + @Test + public void renameOnClientException() throws IOException { + Mockito.when(mClient.listObjects(ArgumentMatchers.any(ListObjectsRequest.class))) + .thenThrow(ServiceException.class); + + boolean result = mOSSUnderFileSystem.renameFile(SRC, DST); + Assert.assertFalse(result); + } + + /** + * Test case for {@link OSSUnderFileSystem#getFolderSuffix()}. + */ + @Test + public void testGetFolderSuffix() { + Assert.assertEquals("_$folder$", mOSSUnderFileSystem.getFolderSuffix()); + } + + /** + * Test case for {@link OSSUnderFileSystem#getObjectListingChunk(ListObjectsRequest)}. + */ + @Test + public void testGetObjectListingChunk() { + // test successful get object listing chunk + Mockito.when(mClient.listObjects(ArgumentMatchers.any(ListObjectsRequest.class))) + .thenReturn(new ObjectListing()); + ListObjectsRequest request = new ListObjectsRequest(); + GenericResult result = mOSSUnderFileSystem.getObjectListingChunk(request); + Assert.assertTrue(result instanceof ObjectListing); + } + + /** + * Test case for {@link OSSUnderFileSystem#isDirectory(String)}. + */ + @Test + public void testIsDirectory() throws IOException { + Assert.assertTrue(mOSSUnderFileSystem.isDirectory("/")); + } + + /** + * Test case for {@link OSSUnderFileSystem#openPositionRead(String, long)}. + */ + @Test + public void testOpenPositionRead() { + PositionReader result = mOSSUnderFileSystem.openPositionRead(KEY, 1L); + Assert.assertTrue(result instanceof OSSPositionReader); + } + + /** + * Test case for {@link OSSUnderFileSystem#getRootKey()}. + */ + @Test + public void testGetRootKey() { + Assert.assertEquals(Constants.HEADER_OSS + BUCKET_NAME, mOSSUnderFileSystem.getRootKey()); + } + + /** + * Test case for {@link OSSUnderFileSystem#openObject(String, OpenOptions, RetryPolicy)}. + */ + @Test + public void testOpenObject() throws IOException { + // test successful open object + Mockito.when(mClient.getObject(ArgumentMatchers.anyString(), ArgumentMatchers.anyString())) + .thenReturn(new OSSObject()); + OpenOptions options = OpenOptions.defaults(); + RetryPolicy retryPolicy = new CountingRetry(1); + InputStream result = mOSSUnderFileSystem.openObject(KEY, options, retryPolicy); + Assert.assertTrue(result instanceof OSSInputStream); + } + } From 2e3b6f67892d114452e643f7cd89e1b03b2bdd35 Mon Sep 17 00:00:00 2001 From: zhaozihao Date: Mon, 14 Aug 2023 13:02:41 +0800 Subject: [PATCH 2/2] Add Unit Test of OSS, OBS and GCS --- .../underfs/gcs/GCSOutputStreamTest.java | 2 - .../underfs/gcs/GCSPositionReaderTest.java | 5 ++- .../gcs/GCSUnderFileSystemFactoryTest.java | 15 +++---- .../underfs/gcs/GCSUnderFileSystemTest.java | 39 ++++++++++--------- .../obs/OBSUnderFileSystemFactoryTest.java | 4 +- .../underfs/obs/OBSUnderFileSystemTest.java | 8 +++- .../oss/OSSUnderFileSystemFactoryTest.java | 4 +- .../underfs/oss/OSSUnderFileSystemTest.java | 1 - 8 files changed, 41 insertions(+), 37 deletions(-) diff --git a/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSOutputStreamTest.java b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSOutputStreamTest.java index 28ed9a9b8b14..814b6bb0430b 100644 --- a/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSOutputStreamTest.java +++ b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSOutputStreamTest.java @@ -18,7 +18,6 @@ import alluxio.conf.Configuration; import alluxio.conf.PropertyKey; -import org.jclouds.s3.domain.ObjectMetadata; import org.jets3t.service.ServiceException; import org.jets3t.service.impl.rest.httpclient.GoogleStorageService; import org.jets3t.service.model.GSObject; @@ -38,7 +37,6 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.security.DigestOutputStream; diff --git a/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSPositionReaderTest.java b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSPositionReaderTest.java index b98a385e572a..f7bdda63b28e 100644 --- a/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSPositionReaderTest.java +++ b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSPositionReaderTest.java @@ -59,8 +59,9 @@ public void before() throws Exception { public void openObjectInputStream() throws Exception { GSObject object = Mockito.mock(GSObject.class); BufferedInputStream objectInputStream = Mockito.mock(BufferedInputStream.class); - Mockito.when(mClient.getObject(Mockito.anyString(), Mockito.anyString(), Mockito.any(), - Mockito.any(),Mockito.any(),Mockito.any(),Mockito.any(),Mockito.any())).thenReturn(object); + Mockito.when(mClient.getObject(Mockito.anyString(), Mockito.anyString(), + Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any(), Mockito.any())).thenReturn(object); Mockito.when(object.getDataInputStream()).thenReturn(objectInputStream); // test successful open object input stream diff --git a/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemFactoryTest.java b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemFactoryTest.java index a15c54fcad8b..17423ccf37b7 100644 --- a/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemFactoryTest.java +++ b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemFactoryTest.java @@ -11,6 +11,8 @@ package alluxio.underfs.gcs; +import static org.junit.Assert.assertFalse; + import alluxio.conf.AlluxioConfiguration; import alluxio.conf.Configuration; import alluxio.conf.PropertyKey; @@ -19,13 +21,10 @@ import alluxio.underfs.UnderFileSystemFactory; import alluxio.underfs.UnderFileSystemFactoryRegistry; -import org.jets3t.service.ServiceException; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertFalse; - /** * Unit tests for the {@link GCSUnderFileSystemFactory}. */ @@ -42,7 +41,6 @@ public final class GCSUnderFileSystemFactoryTest { private UnderFileSystemConfiguration mConf; private UnderFileSystemFactory mFactory; - @Before public void setUp() { Configuration.set(PropertyKey.GCS_ACCESS_KEY, mAccessKey); @@ -94,13 +92,13 @@ public void createInstanceWithoutCredentials() { mFactory.create(mPath, mConf); } catch (Exception e) { Assert.assertTrue(e instanceof RuntimeException); - Assert.assertTrue(e.getMessage().contains("GCS credentials or version not available, " + - "cannot create GCS Under File System.")); + Assert.assertTrue(e.getMessage().contains("GCS credentials or version not available, " + + "cannot create GCS Under File System.")); } Exception e = Assert.assertThrows(RuntimeException.class, () -> mFactory.create( mPath, mConf)); - Assert.assertTrue(e.getMessage().contains("GCS credentials or version not available, " + - "cannot create GCS Under File System.")); + Assert.assertTrue(e.getMessage().contains("GCS credentials or version not available, " + + "cannot create GCS Under File System.")); } /** @@ -113,5 +111,4 @@ public void supportsPath() { assertFalse(mFactory.supportsPath("Invalid_Path")); assertFalse(mFactory.supportsPath("hdfs://test-bucket/path")); } - } diff --git a/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemTest.java b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemTest.java index 1f5996ef6917..bb40229f1041 100644 --- a/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemTest.java +++ b/dora/underfs/gcs/src/test/java/alluxio/underfs/gcs/GCSUnderFileSystemTest.java @@ -12,7 +12,6 @@ package alluxio.underfs.gcs; import static org.junit.Assert.assertFalse; -import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -24,8 +23,8 @@ import alluxio.retry.RetryPolicy; import alluxio.underfs.UnderFileSystemConfiguration; import alluxio.underfs.options.DeleteOptions; - import alluxio.underfs.options.OpenOptions; + import org.jets3t.service.ServiceException; import org.jets3t.service.StorageObjectsChunk; import org.jets3t.service.impl.rest.httpclient.GoogleStorageService; @@ -80,14 +79,16 @@ public void getUnderFSType() { @Test public void testCopyObject() throws ServiceException { // test successful copy object - when(mClient.copyObject(anyString(), anyString(), - anyString(), any(StorageObject.class), anyBoolean())).thenReturn(null); + when(mClient.copyObject(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), ArgumentMatchers.any(StorageObject.class), + ArgumentMatchers.anyBoolean())).thenReturn(null); boolean result = mGCSUnderFileSystem.copyObject(SRC, DST); Assert.assertTrue(result); // test copy object exception - Mockito.when(mClient.copyObject(anyString(), anyString(), - anyString(), any(StorageObject.class), anyBoolean())).thenThrow(ServiceException.class); + Mockito.when(mClient.copyObject(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), ArgumentMatchers.any(StorageObject.class), + ArgumentMatchers.anyBoolean())).thenThrow(ServiceException.class); try { mGCSUnderFileSystem.copyObject(SRC, DST); } catch (Exception e) { @@ -102,13 +103,13 @@ public void testCopyObject() throws ServiceException { public void testCreateEmptyObject() throws ServiceException { // test successful create empty object - Mockito.when(mClient.putObject(anyString(), + Mockito.when(mClient.putObject(ArgumentMatchers.anyString(), ArgumentMatchers.any(GSObject.class))).thenReturn(null); boolean result = mGCSUnderFileSystem.createEmptyObject(KEY); Assert.assertTrue(result); // test create empty object exception - Mockito.when(mClient.putObject(anyString(), + Mockito.when(mClient.putObject(ArgumentMatchers.anyString(), ArgumentMatchers.any(GSObject.class))).thenThrow(ServiceException.class); try { mGCSUnderFileSystem.createEmptyObject(KEY); @@ -123,7 +124,7 @@ public void testCreateEmptyObject() throws ServiceException { @Test public void testCreateObject() throws IOException, ServiceException { // test successful create object - Mockito.when(mClient.putObject(anyString(), + Mockito.when(mClient.putObject(ArgumentMatchers.anyString(), ArgumentMatchers.any(GSObject.class))).thenReturn(null); OutputStream result = mGCSUnderFileSystem.createObject(KEY); Assert.assertTrue(result instanceof GCSOutputStream); @@ -134,8 +135,8 @@ public void testCreateObject() throws IOException, ServiceException { */ @Test public void deleteNonRecursiveOnServiceException() throws IOException, ServiceException { - when(mClient.listObjectsChunked(anyString(), anyString(), - anyString(), ArgumentMatchers.anyLong(), anyString())) + when(mClient.listObjectsChunked(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyLong(), ArgumentMatchers.anyString())) .thenThrow(ServiceException.class); boolean result = mGCSUnderFileSystem.deleteDirectory(PATH, @@ -148,8 +149,8 @@ public void deleteNonRecursiveOnServiceException() throws IOException, ServiceEx */ @Test public void deleteRecursiveOnServiceException() throws IOException, ServiceException { - when(mClient.listObjectsChunked(anyString(), anyString(), - anyString(), ArgumentMatchers.anyLong(), anyString())) + when(mClient.listObjectsChunked(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyLong(), ArgumentMatchers.anyString())) .thenThrow(ServiceException.class); boolean result = mGCSUnderFileSystem.deleteDirectory(PATH, @@ -162,13 +163,14 @@ public void deleteRecursiveOnServiceException() throws IOException, ServiceExcep */ @Test public void renameOnServiceException() throws IOException, ServiceException { - when(mClient.listObjectsChunked(anyString(), anyString(), - anyString(), ArgumentMatchers.anyLong(), anyString())) + when(mClient.listObjectsChunked(ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyLong(), ArgumentMatchers.anyString())) .thenThrow(ServiceException.class); boolean result = mGCSUnderFileSystem.renameFile(SRC, DST); assertFalse(result); } + /** * Test case for {@link GCSUnderFileSystem#getFolderSuffix()}. */ @@ -183,8 +185,9 @@ public void testGetFolderSuffix() { @Test public void testGetObjectListingChunk() throws ServiceException { // test successful get object listing chunk - Mockito.when(mClient.listObjectsChunked(anyString(), anyString(), - anyString(), anyLong(), anyString())).thenReturn(new StorageObjectsChunk( + Mockito.when(mClient.listObjectsChunked(ArgumentMatchers.anyString(), + ArgumentMatchers.anyString(), ArgumentMatchers.anyString(), ArgumentMatchers.anyLong(), + ArgumentMatchers.anyString())).thenReturn(new StorageObjectsChunk( null, null, null, null, null)); String delimiter = "/"; String prefix = ""; @@ -223,7 +226,7 @@ public void testGetRootKey() { @Test public void testOpenObject() throws IOException, ServiceException { // test successful open object - Mockito.when(mClient.getObject(anyString(), anyString())) + Mockito.when(mClient.getObject(ArgumentMatchers.anyString(), ArgumentMatchers.anyString())) .thenReturn(new GSObject()); OpenOptions options = OpenOptions.defaults(); RetryPolicy retryPolicy = new CountingRetry(1); diff --git a/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemFactoryTest.java b/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemFactoryTest.java index 2f18ac10c38b..f760179f9360 100644 --- a/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemFactoryTest.java +++ b/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemFactoryTest.java @@ -11,6 +11,8 @@ package alluxio.underfs.obs; +import static org.junit.Assert.assertFalse; + import alluxio.conf.AlluxioConfiguration; import alluxio.conf.Configuration; import alluxio.conf.PropertyKey; @@ -23,8 +25,6 @@ import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertFalse; - /** * Unit tests for the {@link OBSUnderFileSystemFactory}. */ diff --git a/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemTest.java b/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemTest.java index 657efa081642..d704cdea8f34 100644 --- a/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemTest.java +++ b/dora/underfs/obs/src/test/java/alluxio/underfs/obs/OBSUnderFileSystemTest.java @@ -23,7 +23,13 @@ import com.obs.services.ObsClient; import com.obs.services.exception.ObsException; -import com.obs.services.model.*; +import com.obs.services.model.DeleteObjectsRequest; +import com.obs.services.model.DeleteObjectsResult; +import com.obs.services.model.HeaderResponse; +import com.obs.services.model.ListObjectsRequest; +import com.obs.services.model.ObjectListing; +import com.obs.services.model.ObjectMetadata; +import com.obs.services.model.ObsObject; import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemFactoryTest.java b/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemFactoryTest.java index e900b8d69a7a..0e9127b27b9a 100644 --- a/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemFactoryTest.java +++ b/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemFactoryTest.java @@ -11,6 +11,8 @@ package alluxio.underfs.oss; +import static org.junit.Assert.assertFalse; + import alluxio.conf.AlluxioConfiguration; import alluxio.conf.Configuration; import alluxio.conf.PropertyKey; @@ -23,8 +25,6 @@ import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertFalse; - /** * Unit tests for the {@link OSSUnderFileSystemFactory}. */ diff --git a/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemTest.java b/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemTest.java index 11a4011d0823..03abc9f0e1a3 100644 --- a/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemTest.java +++ b/dora/underfs/oss/src/test/java/alluxio/underfs/oss/OSSUnderFileSystemTest.java @@ -269,5 +269,4 @@ public void testOpenObject() throws IOException { InputStream result = mOSSUnderFileSystem.openObject(KEY, options, retryPolicy); Assert.assertTrue(result instanceof OSSInputStream); } - }