Fix linting

caraml-dev · Sep 26, 2024 · a04f6f7 · a04f6f7
1 parent 58c09fc
commit a04f6f7
Show file tree

Hide file tree

Showing 5 changed files with 28 additions and 17 deletions.
diff --git a/...store-serving/src/test/java/dev/caraml/serving/store/bigtable/GenericHbase2Container.java b/...store-serving/src/test/java/dev/caraml/serving/store/bigtable/GenericHbase2Container.java
@@ -3,7 +3,6 @@
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.time.Duration;
-import java.util.Arrays;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.testcontainers.containers.GenericContainer;
@@ -40,10 +39,10 @@ public GenericHbase2Container() {
 
     withEnv("HBASE_MASTER_PORT", Integer.toString(masterPort));
     withEnv("HBASE_REGION_PORT", Integer.toString(regionPort));
-//    setPortBindings(
-//        Arrays.asList(
-//            String.format("%d:%d", masterPort, masterPort),
-//            String.format("%d:%d", regionPort, regionPort)));
+    //    setPortBindings(
+    //        Arrays.asList(
+    //            String.format("%d:%d", masterPort, masterPort),
+    //            String.format("%d:%d", regionPort, regionPort)));
 
     // Set network mode to host
     withNetworkMode("host");

diff --git a/...ore-serving/src/test/java/dev/caraml/serving/store/bigtable/HbaseOnlineRetrieverTest.java b/...ore-serving/src/test/java/dev/caraml/serving/store/bigtable/HbaseOnlineRetrieverTest.java
@@ -38,10 +38,11 @@ public class HbaseOnlineRetrieverTest {
 
   @BeforeAll
   public static void setup() throws IOException {
-//    hbaseConfiguration.set("hbase.zookeeper.quorum", hbase.getHost());
-//    hbaseConfiguration.set("hbase.zookeeper.property.clientPort", hbase.getMappedPort(2181).toString());
-//    hbaseConfiguration.set("hbase.zookeeper.property.clientPort", "2181");
-//    hbaseClient = ConnectionFactory.createConnection(hbaseConfiguration);
+    //    hbaseConfiguration.set("hbase.zookeeper.quorum", hbase.getHost());
+    //    hbaseConfiguration.set("hbase.zookeeper.property.clientPort",
+    // hbase.getMappedPort(2181).toString());
+    //    hbaseConfiguration.set("hbase.zookeeper.property.clientPort", "2181");
+    //    hbaseClient = ConnectionFactory.createConnection(hbaseConfiguration);
     hbaseClient = ConnectionFactory.createConnection(hbase.hbase2Configuration);
     admin = (HBaseAdmin) hbaseClient.getAdmin();
     ingestData();

diff --git a/caraml-store-spark/src/main/scala/dev/caraml/spark/BasePipeline.scala b/caraml-store-spark/src/main/scala/dev/caraml/spark/BasePipeline.scala
@@ -37,7 +37,10 @@ object BasePipeline {
         conf
           .set("spark.hbase.zookeeper.quorum", zookeeperQuorum)
           .set("spark.hbase.zookeeper.port", zookeeperPort.toString)
-          .set("spark.hbase.properties.regionSplitPolicyClassName", hbaseProperties.regionSplitPolicy)
+          .set(
+            "spark.hbase.properties.regionSplitPolicyClassName",
+            hbaseProperties.regionSplitPolicy
+          )
           .set("spark.hbase.properties.compressionAlgorithm", hbaseProperties.compressionAlgorithm)
     }
 

diff --git a/caraml-store-spark/src/main/scala/dev/caraml/spark/IngestionJobConfig.scala b/caraml-store-spark/src/main/scala/dev/caraml/spark/IngestionJobConfig.scala
@@ -26,9 +26,17 @@ case class RedisWriteProperties(
     enableRateLimit: Boolean = false,
     ratePerSecondLimit: Int = 50000
 )
-case class BigTableConfig(projectId: String, instanceId: String)    extends StoreConfig
-case class HBaseConfig(zookeeperQuorum: String, zookeeperPort: Int, hbaseProperties: HBaseProperties = HBaseProperties()) extends StoreConfig
-case class HBaseProperties(regionSplitPolicy: String = "org.apache.hadoop.hbase.regionserver.IncreasingToUpperBoundRegionSplitPolicy", compressionAlgorithm: String = "ZSTD")
+case class BigTableConfig(projectId: String, instanceId: String) extends StoreConfig
+case class HBaseConfig(
+    zookeeperQuorum: String,
+    zookeeperPort: Int,
+    hbaseProperties: HBaseProperties = HBaseProperties()
+) extends StoreConfig
+case class HBaseProperties(
+    regionSplitPolicy: String =
+      "org.apache.hadoop.hbase.regionserver.IncreasingToUpperBoundRegionSplitPolicy",
+    compressionAlgorithm: String = "ZSTD"
+)
 
 sealed trait MetricConfig
 

diff --git a/caraml-store-spark/src/main/scala/dev/caraml/spark/stores/bigtable/HbaseSinkRelation.scala b/caraml-store-spark/src/main/scala/dev/caraml/spark/stores/bigtable/HbaseSinkRelation.scala
@@ -42,11 +42,11 @@ class HbaseSinkRelation(
       }
       featuresCFBuilder.setMaxVersions(1)
       sqlContext.getConf("spark.hbase.properties.compressionAlgorithm") match {
-        case "ZSTD" => featuresCFBuilder.setCompressionType(Compression.Algorithm.ZSTD)
-        case "GZ" => featuresCFBuilder.setCompressionType(Compression.Algorithm.GZ)
-        case "LZ4" => featuresCFBuilder.setCompressionType(Compression.Algorithm.LZ4)
+        case "ZSTD"   => featuresCFBuilder.setCompressionType(Compression.Algorithm.ZSTD)
+        case "GZ"     => featuresCFBuilder.setCompressionType(Compression.Algorithm.GZ)
+        case "LZ4"    => featuresCFBuilder.setCompressionType(Compression.Algorithm.LZ4)
         case "SNAPPY" => featuresCFBuilder.setCompressionType(Compression.Algorithm.SNAPPY)
-        case _ => featuresCFBuilder.setCompressionType(Compression.Algorithm.NONE)
+        case _        => featuresCFBuilder.setCompressionType(Compression.Algorithm.NONE)
       }
       val featuresCF = featuresCFBuilder.build()