Variant to HBase load: allow to specify hadoop cluster

Use output option to specify hadoop cluster url (issue #11)

hpg-bigdata-app/src/main/java/org/opencb/hpg/bigdata/app/cli/ConvertCommandExecutor.java

@@ -43,6 +43,8 @@
 import java.nio.ByteBuffer;
 import java.nio.CharBuffer;
 import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -492,8 +494,12 @@ private void cram2avro(String input, String output, String codecName) throws IOE
 
     private void variant2hbase(String input, String output) throws Exception {
 		Variant2HbaseMR mr = new Variant2HbaseMR();
-		String[] args = new String[]{"-i",input,"-t","VariantLoad"};
-		int run = ToolRunner.run(mr, args);
+		List<String> args = new ArrayList<String>(Arrays.asList(new String[]{"-i",input,"-t","VariantLoad"}));
+		if(StringUtils.isNotBlank(output)){
+			args.add("-o");
+			args.add(output);
+		}
+		int run = ToolRunner.run(mr, args.toArray(new String[0]));
 		if(run != 0)
 			throw new IllegalStateException(String.format("Variant 2 HBase finished with %s !", run));
 	}

hpg-bigdata-tools/src/main/java/org/opencb/hpg/bigdata/tools/converters/mr/Variant2HbaseMR.java

@@ -10,6 +10,7 @@
 import org.apache.avro.mapreduce.AvroJob;
 import org.apache.avro.mapreduce.AvroKeyInputFormat;
 import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -155,11 +156,14 @@ public Configuration getConf() {
 	public int run(String[] args) throws Exception {
 		String tablename = "test_table";
 		String inputfile = null;
+		String output = null;
 		for(int i = 0; i < args.length; ++i){
 			if(args[i] == "-t")
 				tablename = args[++i];
 			if(args[i] == "-i")
 				inputfile = args[++i];
+			if(args[i] == "-o")
+				output = args[++i];
 		}
 
 	    setConf(HBaseConfiguration.addHbaseResources(getConf()));
@@ -172,19 +176,27 @@ public int run(String[] args) throws Exception {
 		AvroJob.setInputKeySchema(job, Variant.getClassSchema());
 		FileInputFormat.setInputPaths(job, new Path(inputfile));
 		job.setInputFormatClass(AvroKeyInputFormat.class);
-
 
 		// output -> Hbase
 		TableMapReduceUtil.initTableReducerJob(tablename, null, job);
 		job.setNumReduceTasks(0); // Write to table directory
+		if(StringUtils.isNotBlank(output)){
+			Configuration conf = getConf();
+			conf.set("hbase.zookeeper.quorum", output);
+			conf.set("hbase.master", output+":60000");
+			setConf(conf);
+		}
 
 		// mapper
 		job.setMapperClass(Variant2HbaseMR.class);
 
 		// create Table if needed
 		createTableIfNeeded(tablename);
-
-		return job.waitForCompletion(true)?0:1;
+		long start = System.currentTimeMillis();
+		boolean completed = job.waitForCompletion(true);
+		long end = System.currentTimeMillis();
+		getLog().info(String.format("Loading run for %s ms!", (end-start)));
+		return completed?0:1;
 	}
 
 	/**

pom.xml

@@ -144,7 +144,7 @@
 				<artifactId>hadoop-common</artifactId>
 				<version>${hadoop.version}</version>
 			</dependency>
-			
+
 			<!-- General dependencies -->
 			<dependency>
 				<groupId>com.google.guava</groupId>
@@ -172,6 +172,13 @@
 				<version>4.12</version>
 				<scope>test</scope>
 			</dependency>
+			<dependency>
+				<groupId>jdk.tools</groupId>
+				<artifactId>jdk.tools</artifactId>
+				<version>1.6</version>
+				<scope>system</scope>
+				<systemPath>${java.home}/lib/tools.jar</systemPath>
+			</dependency>
 		</dependencies>
 	</dependencyManagement>