Migration to HDP v3.1 (78)

flipkart-incubator · Apr 8, 2019 · 5297b63 · 5297b63
1 parent 06d9b6a
commit 5297b63
Show file tree

Hide file tree

Showing 20 changed files with 96 additions and 429 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,4 +3,6 @@
 /output/
 /out/
 /target/
-.DS_Store
+.DS_Store
+*.txt
+*.rdb
diff --git a/README.md b/README.md
@@ -3,9 +3,8 @@
 ## Introduction
 An ultra-light-weight HBase ORM library that enables
 
-1. reading from and/or writing to HBase tables in Hadoop MapReduce jobs
-2. object-oriented access of HBase rows ([Data Access Object](https://en.wikipedia.org/wiki/Data_access_object))
-3. writing high-quality test cases for classes that interact with HBase
+1. object-oriented access of HBase rows (Data Access Object) with minimal code and good testability
+2. reading from and/or writing to HBase tables in Hadoop MapReduce jobs
 
 
 ## Usage
@@ -262,7 +261,7 @@ Add below entry within the `dependencies` section of your `pom.xml`:
 	<version>1.11</version>
 </dependency>
 ```
-See artifact details: [com.flipkart:hbase-object-mapper on **Maven Central**](http://search.maven.org/#search%7Cgav%7C1%7Cg%3A%22com.flipkart%22%20AND%20a%3A%22hbase-object-mapper%22) or
+See artifact details: [com.flipkart:hbase-object-mapper on **Maven Central**](https://search.maven.org/search?q=g:com.flipkart%20AND%20a:hbase-object-mapper&core=gav) or
 [com.flipkart:hbase-object-mapper on **MVN Repository**](https://mvnrepository.com/artifact/com.flipkart/hbase-object-mapper).
 ## How to build?
 To build this project, follow below simple steps:

diff --git a/pom.xml b/pom.xml
@@ -5,14 +5,13 @@
     <name>HBase ORM</name>
     <description>
         An ultra-light-weight HBase ORM library that enables
-        [1] object-oriented access of HBase rows (Data Access Object)
+        [1] object-oriented access of HBase rows (Data Access Object) with minimal code and good testability
         [2] reading from and/or writing to HBase tables in Hadoop MapReduce jobs
-        [3] writing high-quality test cases for classes that interact with HBase
     </description>
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.flipkart</groupId>
     <artifactId>hbase-object-mapper</artifactId>
-    <version>1.11</version>
+    <version>1.11-hdp31-78</version>
     <url>https://flipkart-incubator.github.io/hbase-orm/</url>
     <scm>
         <url>https://github.com/flipkart-incubator/hbase-orm/</url>
@@ -45,8 +44,9 @@
     </repositories>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <version.hadoop>2.7.1.2.4.0.0-169</version.hadoop>
-        <version.hbase>1.1.2.2.4.0.0-169</version.hbase>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <version.hadoop>3.1.1.3.1.0.0-78</version.hadoop>
+        <version.hbase>2.0.2.3.1.0.0-78</version.hbase>
     </properties>
     <distributionManagement>
         <repository>
@@ -66,11 +66,16 @@
             <artifactId>jackson-databind</artifactId>
             <version>2.8.11.3</version>
         </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>19.0</version>
+        </dependency>
         <!-- test dependencies -->
         <dependency>
             <groupId>org.projectlombok</groupId>
             <artifactId>lombok</artifactId>
-            <version>1.16.16</version>
+            <version>1.18.4</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -90,24 +95,23 @@
             <artifactId>hbase-testing-util</artifactId>
             <version>${version.hbase}</version>
             <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>junit</groupId>
+                    <artifactId>junit</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
-            <groupId>org.apache.mrunit</groupId>
-            <artifactId>mrunit</artifactId>
-            <version>1.1.0</version>
-            <classifier>hadoop2</classifier>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <version>5.3.2</version>
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>org.javatuples</groupId>
-            <artifactId>javatuples</artifactId>
-            <version>1.2</version>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>2.23.4</version>
             <scope>test</scope>
         </dependency>
     </dependencies>
@@ -116,10 +120,10 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
-                <version>3.6.1</version>
+                <version>3.8.0</version>
                 <configuration>
-                    <source>1.7</source>
-                    <target>1.7</target>
+                    <source>1.8</source>
+                    <target>1.8</target>
                     <compilerArgument>-Xlint:all</compilerArgument>
                 </configuration>
             </plugin>

diff --git a/src/main/java/com/flipkart/hbaseobjectmapper/AbstractHBDAO.java b/src/main/java/com/flipkart/hbaseobjectmapper/AbstractHBDAO.java
@@ -147,7 +147,7 @@ protected AbstractHBDAO(Configuration configuration) throws IOException {
      * @throws IOException When HBase call fails
      */
     public T get(R rowKey, int numVersionsToFetch) throws IOException {
-        Result result = this.table.get(new Get(toBytes(rowKey)).setMaxVersions(numVersionsToFetch));
+        Result result = this.table.get(new Get(toBytes(rowKey)).readVersions(numVersionsToFetch));
         return hbObjectMapper.readValue(rowKey, result, hbRecordClass);
     }
 
@@ -214,7 +214,7 @@ public List<T> getOnGets(List<Get> gets) throws IOException {
     public T[] get(R[] rowKeys, int numVersionsToFetch) throws IOException {
         List<Get> gets = new ArrayList<>(rowKeys.length);
         for (R rowKey : rowKeys) {
-            gets.add(new Get(toBytes(rowKey)).setMaxVersions(numVersionsToFetch));
+            gets.add(new Get(toBytes(rowKey)).readVersions(numVersionsToFetch));
         }
         Result[] results = this.table.get(gets);
         @SuppressWarnings("unchecked") T[] records = (T[]) Array.newInstance(hbRecordClass, rowKeys.length);
@@ -246,7 +246,7 @@ public T[] get(R[] rowKeys) throws IOException {
     public List<T> get(List<R> rowKeys, int numVersionsToFetch) throws IOException {
         List<Get> gets = new ArrayList<>(rowKeys.size());
         for (R rowKey : rowKeys) {
-            gets.add(new Get(toBytes(rowKey)).setMaxVersions(numVersionsToFetch));
+            gets.add(new Get(toBytes(rowKey)).readVersions(numVersionsToFetch));
         }
         Result[] results = this.table.get(gets);
         List<T> records = new ArrayList<>(rowKeys.size());
@@ -277,7 +277,7 @@ public List<T> get(List<R> rowKeys) throws IOException {
      * @throws IOException When HBase call fails
      */
     public List<T> get(R startRowKey, R endRowKey, int numVersionsToFetch) throws IOException {
-        Scan scan = new Scan(toBytes(startRowKey), toBytes(endRowKey)).setMaxVersions(numVersionsToFetch);
+        Scan scan = new Scan().withStartRow(toBytes(startRowKey)).withStopRow(toBytes(endRowKey)).readVersions(numVersionsToFetch);
         ResultScanner scanner = table.getScanner(scan);
         List<T> records = new ArrayList<>();
         for (Result result : scanner) {
@@ -312,7 +312,7 @@ public long increment(R rowKey, String fieldName, long amount) throws IOExceptio
      * Increments field by specified amount
      *
      * @param rowKey     Row key of the record whose column needs to be incremented
-     * @param fieldName Field that needs to be incremented (this must be of {@link Long} type)
+     * @param fieldName  Field that needs to be incremented (this must be of {@link Long} type)
      * @param amount     Amount by which the HBase column needs to be incremented
      * @param durability The persistence guarantee for this increment (see {@link Durability})
      * @return The new value, post increment
@@ -337,9 +337,9 @@ public Increment getIncrement(R rowKey) {
      * Performs HBase {@link Table#increment} on the given {@link Increment} object <br>
      * <br>
      * <b>Note</b>: <ul>
-     *     <li>You may construct {@link Increment} object using the {@link #getIncrement(Serializable) getIncrement} method</li>
-     *     <li>Unlike the {@link #increment(Serializable, String, long)} methods, this method skips some validations (hence, be cautious)</li>
-     *     </ul>
+     * <li>You may construct {@link Increment} object using the {@link #getIncrement(Serializable) getIncrement} method</li>
+     * <li>Unlike the {@link #increment(Serializable, String, long)} methods, this method skips some validations (hence, be cautious)</li>
+     * </ul>
      *
      * @param increment HBase Increment object
      * @return <b>Partial object</b> containing (only) values that were incremented
@@ -570,9 +570,9 @@ private Map<R, Object> toSingleVersioned(Map<R, NavigableMap<Long, Object>> mult
     public NavigableMap<R, NavigableMap<Long, Object>> fetchFieldValues(R startRowKey, R endRowKey, String fieldName, int numVersionsToFetch) throws IOException {
         Field field = getField(fieldName);
         WrappedHBColumn hbColumn = new WrappedHBColumn(field, true);
-        Scan scan = new Scan(toBytes(startRowKey), toBytes(endRowKey));
+        Scan scan = new Scan().withStartRow(toBytes(startRowKey)).withStopRow(toBytes(endRowKey));
         scan.addColumn(hbColumn.familyBytes(), hbColumn.columnBytes());
-        scan.setMaxVersions(numVersionsToFetch);
+        scan.readVersions(numVersionsToFetch);
         ResultScanner scanner = table.getScanner(scan);
         NavigableMap<R, NavigableMap<Long, Object>> map = new TreeMap<>();
         for (Result result : scanner) {
@@ -609,7 +609,7 @@ public Map<R, NavigableMap<Long, Object>> fetchFieldValues(R[] rowKeys, String f
         List<Get> gets = new ArrayList<>(rowKeys.length);
         for (R rowKey : rowKeys) {
             Get get = new Get(toBytes(rowKey));
-            get.setMaxVersions(numVersionsToFetch);
+            get.readVersions(numVersionsToFetch);
             get.addColumn(hbColumn.familyBytes(), hbColumn.columnBytes());
             gets.add(get);
         }

diff --git a/src/main/java/com/flipkart/hbaseobjectmapper/HBObjectMapper.java b/src/main/java/com/flipkart/hbaseobjectmapper/HBObjectMapper.java
@@ -4,8 +4,8 @@
 import com.flipkart.hbaseobjectmapper.codec.Codec;
 import com.flipkart.hbaseobjectmapper.codec.exceptions.DeserializationException;
 import com.flipkart.hbaseobjectmapper.codec.exceptions.SerializationException;
-import com.flipkart.hbaseobjectmapper.exceptions.*;
 import com.flipkart.hbaseobjectmapper.exceptions.InternalError;
+import com.flipkart.hbaseobjectmapper.exceptions.*;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HConstants;