Skip to content

[MINOR][IMPROVEMENT][STORAGE] Fix storage module code style #131

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
roryqi merged 1 commit into from
Aug 5, 2022
Merged

[MINOR][IMPROVEMENT][STORAGE] Fix storage module code style #131

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions storage/src/test/java/org/apache/uniffle/storage/HdfsShuffleHandlerTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,16 @@

package org.apache.uniffle.storage;

import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicLong;

import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;

import org.apache.uniffle.common.BufferSegment;
import org.apache.uniffle.common.ShuffleDataResult;
import org.apache.uniffle.common.ShufflePartitionedBlock;
Expand All @@ -30,17 +36,13 @@
import org.apache.uniffle.storage.handler.impl.HdfsFileReader;
import org.apache.uniffle.storage.handler.impl.HdfsFileWriter;
import org.apache.uniffle.storage.handler.impl.HdfsShuffleWriteHandler;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;

import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;

public class HdfsShuffleHandlerTestBase extends HdfsTestBase {

private final static AtomicLong ATOMIC_LONG = new AtomicLong(0);
private static final AtomicLong ATOMIC_LONG = new AtomicLong(0);

protected void writeTestData(
HdfsShuffleWriteHandler writeHandler,
Expand Down
23 changes: 12 additions & 11 deletions storage/src/test/java/org/apache/uniffle/storage/common/LocalStorageTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,22 @@

package org.apache.uniffle.storage.common;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.File;
import java.io.IOException;
import java.util.List;

import com.google.common.collect.Lists;
import org.apache.uniffle.common.util.RssUtils;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.roaringbitmap.RoaringBitmap;

import java.io.File;
import java.io.IOException;
import java.util.List;
import org.apache.uniffle.common.util.RssUtils;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;


public class LocalStorageTest {
Expand Down Expand Up @@ -74,8 +75,8 @@ public void canWriteTest() {
@Test
public void removeResourcesTest() throws Exception {
LocalStorage item = prepareDiskItem();
String key1 = RssUtils.generateShuffleKey("1", 1);
String key2 = RssUtils.generateShuffleKey("1", 2);
final String key1 = RssUtils.generateShuffleKey("1", 1);
final String key2 = RssUtils.generateShuffleKey("1", 2);
item.removeResources(key1);
assertEquals(50L, item.getMetaData().getDiskSize().get());
assertEquals(0L, item.getMetaData().getShuffleSize(key1));
Expand All @@ -84,7 +85,7 @@ public void removeResourcesTest() throws Exception {
}

private LocalStorage prepareDiskItem() {
LocalStorage item = LocalStorage.newBuilder().basePath(testBaseDir.getAbsolutePath())
final LocalStorage item = LocalStorage.newBuilder().basePath(testBaseDir.getAbsolutePath())
.cleanupThreshold(50)
.highWaterMarkOfWrite(95)
.lowWaterMarkOfWrite(80)
Expand Down Expand Up @@ -116,7 +117,7 @@ public void concurrentRemoveResourcesTest() throws Exception {
try {
t.join();
} catch (InterruptedException e) {

// ignore
}
});

Expand Down
1 change: 1 addition & 0 deletions storage/src/test/java/org/apache/uniffle/storage/common/ShuffleFileInfoTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.uniffle.storage.common;

import java.io.File;

import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertFalse;
Expand Down
13 changes: 7 additions & 6 deletions storage/src/test/java/org/apache/uniffle/storage/handler/impl/HdfsClientReadHandlerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,18 @@

package org.apache.uniffle.storage.handler.impl;

import java.util.Map;
import java.util.Random;
import java.util.Set;

import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.uniffle.common.BufferSegment;
import org.apache.uniffle.common.ShuffleDataResult;
import org.apache.uniffle.storage.HdfsShuffleHandlerTestBase;
import org.junit.jupiter.api.Test;
import org.roaringbitmap.longlong.Roaring64NavigableMap;

import java.util.Map;
import java.util.Random;
import java.util.Set;
import org.apache.uniffle.common.BufferSegment;
import org.apache.uniffle.common.ShuffleDataResult;
import org.apache.uniffle.storage.HdfsShuffleHandlerTestBase;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
Expand Down
9 changes: 5 additions & 4 deletions storage/src/test/java/org/apache/uniffle/storage/handler/impl/HdfsFileReaderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,17 @@

package org.apache.uniffle.storage.handler.impl;

import org.apache.uniffle.common.util.ChecksumUtils;
import org.apache.uniffle.storage.HdfsTestBase;
import org.apache.uniffle.storage.common.FileBasedShuffleSegment;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Random;

import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;

import org.apache.uniffle.common.util.ChecksumUtils;
import org.apache.uniffle.storage.HdfsTestBase;
import org.apache.uniffle.storage.common.FileBasedShuffleSegment;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
Expand Down
6 changes: 4 additions & 2 deletions storage/src/test/java/org/apache/uniffle/storage/handler/impl/HdfsFileWriterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,19 @@

package org.apache.uniffle.storage.handler.impl;

import org.apache.uniffle.storage.HdfsTestBase;
import org.apache.uniffle.storage.common.FileBasedShuffleSegment;
import java.io.EOFException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Random;

import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;

import org.apache.uniffle.storage.HdfsTestBase;
import org.apache.uniffle.storage.common.FileBasedShuffleSegment;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
Expand Down
23 changes: 12 additions & 11 deletions storage/src/test/java/org/apache/uniffle/storage/handler/impl/HdfsHandlerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,27 @@

package org.apache.uniffle.storage.handler.impl;

import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
import java.util.Set;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;
import org.roaringbitmap.longlong.Roaring64NavigableMap;

import org.apache.uniffle.common.BufferSegment;
import org.apache.uniffle.common.ShuffleDataResult;
import org.apache.uniffle.common.ShufflePartitionedBlock;
import org.apache.uniffle.storage.HdfsTestBase;
import org.apache.uniffle.storage.common.FileBasedShuffleSegment;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;
import org.roaringbitmap.longlong.Roaring64NavigableMap;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertTrue;

public class HdfsHandlerTest extends HdfsTestBase {

Expand Down
18 changes: 10 additions & 8 deletions ...age/src/test/java/org/apache/uniffle/storage/handler/impl/HdfsShuffleReadHandlerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,23 @@

package org.apache.uniffle.storage.handler.impl;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.fail;
import java.util.Map;
import java.util.Random;
import java.util.Set;

import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.junit.jupiter.api.Test;
import org.roaringbitmap.longlong.Roaring64NavigableMap;

import org.apache.uniffle.common.BufferSegment;
import org.apache.uniffle.common.ShuffleDataResult;
import org.apache.uniffle.storage.HdfsShuffleHandlerTestBase;
import org.apache.uniffle.storage.util.ShuffleStorageUtils;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import org.junit.jupiter.api.Test;
import org.roaringbitmap.longlong.Roaring64NavigableMap;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.fail;

public class HdfsShuffleReadHandlerTest extends HdfsShuffleHandlerTestBase {

Expand Down
38 changes: 20 additions & 18 deletions storage/src/test/java/org/apache/uniffle/storage/handler/impl/LocalFileHandlerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,20 @@

package org.apache.uniffle.storage.handler.impl;

import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import org.junit.jupiter.api.Test;

import org.apache.uniffle.common.BufferSegment;
import org.apache.uniffle.common.ShuffleDataResult;
import org.apache.uniffle.common.ShuffleDataSegment;
Expand All @@ -37,14 +42,11 @@
import org.apache.uniffle.storage.handler.api.ServerReadHandler;
import org.apache.uniffle.storage.handler.api.ShuffleWriteHandler;
import org.apache.uniffle.storage.util.ShuffleStorageUtils;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

public class LocalFileHandlerTest {

Expand All @@ -58,21 +60,21 @@ public void writeTest() throws Exception {
File dataDir2 = new File(tmpDir, "data2");
String[] basePaths = new String[]{dataDir1.getAbsolutePath(),
dataDir2.getAbsolutePath()};
LocalFileWriteHandler writeHandler1 = new LocalFileWriteHandler("appId", 0, 1, 1,
final LocalFileWriteHandler writeHandler1 = new LocalFileWriteHandler("appId", 0, 1, 1,
basePaths[0], "pre");
LocalFileWriteHandler writeHandler2 = new LocalFileWriteHandler("appId", 0, 2, 2,
final LocalFileWriteHandler writeHandler2 = new LocalFileWriteHandler("appId", 0, 2, 2,
basePaths[0], "pre");

String possiblePath1 = ShuffleStorageUtils.getFullShuffleDataFolder(dataDir1.getAbsolutePath(),
ShuffleStorageUtils.getShuffleDataPath("appId", 0, 1, 1));
String possiblePath2 = ShuffleStorageUtils.getFullShuffleDataFolder(dataDir2.getAbsolutePath(),
ShuffleStorageUtils.getShuffleDataPath("appId", 0, 1, 1));
assertTrue(writeHandler1.getBasePath().endsWith(possiblePath1) ||
writeHandler1.getBasePath().endsWith(possiblePath2));
assertTrue(writeHandler1.getBasePath().endsWith(possiblePath1)
|| writeHandler1.getBasePath().endsWith(possiblePath2));

Map<Long, byte[]> expectedData = Maps.newHashMap();
Set<Long> expectedBlockIds1 = Sets.newHashSet();
Set<Long> expectedBlockIds2 = Sets.newHashSet();
final Set<Long> expectedBlockIds1 = Sets.newHashSet();
final Set<Long> expectedBlockIds2 = Sets.newHashSet();

writeTestData(writeHandler1, 1, 32, expectedData, expectedBlockIds1);
writeTestData(writeHandler1, 2, 32, expectedData, expectedBlockIds1);
Expand Down
13 changes: 7 additions & 6 deletions storage/src/test/java/org/apache/uniffle/storage/util/ShuffleHdfsStorageUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,18 @@

package org.apache.uniffle.storage.util;

import org.apache.uniffle.storage.HdfsTestBase;
import org.apache.uniffle.storage.handler.impl.HdfsFileWriter;
import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.util.Random;

import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

import org.apache.uniffle.storage.HdfsTestBase;
import org.apache.uniffle.storage.handler.impl.HdfsFileWriter;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;

Expand Down
13 changes: 7 additions & 6 deletions storage/src/test/java/org/apache/uniffle/storage/util/ShuffleStorageUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,20 @@

package org.apache.uniffle.storage.util;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import java.util.List;
import java.util.Set;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.junit.jupiter.api.Test;

import org.apache.uniffle.common.BufferSegment;
import org.apache.uniffle.storage.common.FileBasedShuffleSegment;
import org.apache.uniffle.storage.handler.impl.DataFileSegment;
import java.util.List;
import java.util.Set;

import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;

public class ShuffleStorageUtilsTest {

Expand Down