Skip to content

Commit

Permalink
HBASE-28616 Remove/Deprecated the rs.* related configuration in Table…
Browse files Browse the repository at this point in the history 
…OutputFormat (#5946)

Signed-off-by: Andrew Purtell <apurtell@apache.org>
Signed-off-by: Pankaj Kumar <pankajkumar@apache.org>
Reviewed-by: Subrat Mishra <subrat.mishra@gmail.com>
  • Loading branch information
@Apache9
Apache9 authored May 29, 2024
1 parent 826fb41 commit 771e1d9
Show file tree
Hide file tree
Showing 5 changed files with 87 additions and 30 deletions.
2 changes: 1 addition & 1 deletion ...rc/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ public int runCopier(String outputDir) throws Exception {
}
job.getConfiguration().setBoolean("mapreduce.map.speculative", false);
job.getConfiguration().setBoolean("mapreduce.reduce.speculative", false);
TableMapReduceUtil.initTableReducerJob(COMMON_TABLE_NAME, null, job, null, null, null, null);
TableMapReduceUtil.initTableReducerJob(COMMON_TABLE_NAME, null, job);
TableMapReduceUtil.addDependencyJars(job);
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
AbstractHBaseTool.class);
Expand Down
3 changes: 1 addition & 2 deletions hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,8 +181,7 @@ public Job createSubmittableJob(String[] args) throws IOException {
}
} else {
initCopyTableMapperReducerJob(job, scan);
TableMapReduceUtil.initTableReducerJob(dstTableName, null, job, null, peerAddress, null,
null);
TableMapReduceUtil.initTableReducerJob(dstTableName, null, job, null, peerAddress);
}

return job;
Expand Down
3 changes: 1 addition & 2 deletions hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,8 +167,7 @@ public Job createSubmittableJob(String[] args) throws IOException {
} else {
// No reducers. Just write straight to table. Call initTableReducerJob
// because it sets up the TableOutputFormat.
TableMapReduceUtil.initTableReducerJob(targetTableName, null, job, null, targetZkCluster,
null, null);
TableMapReduceUtil.initTableReducerJob(targetTableName, null, job, null, targetZkCluster);

// would be nice to add an option for bulk load instead
}
Expand Down
87 changes: 71 additions & 16 deletions hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -549,7 +549,7 @@ public static void initTableReducerJob(String table, Class<? extends TableReduce
*/
public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
Job job, Class partitioner) throws IOException {
initTableReducerJob(table, reducer, job, partitioner, null, null, null);
initTableReducerJob(table, reducer, job, partitioner, null);
}

/**
Expand All @@ -569,15 +569,11 @@ public static void initTableReducerJob(String table, Class<? extends TableReduce
* <code> &lt;hbase.zookeeper.quorum&gt;:&lt;
* hbase.zookeeper.client.port&gt;:&lt;zookeeper.znode.parent&gt;
* </code> such as <code>server,server2,server3:2181:/hbase</code>.
* @param serverClass redefined hbase.regionserver.class
* @param serverImpl redefined hbase.regionserver.impl
* @throws IOException When determining the region count fails.
*/
public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl)
throws IOException {
initTableReducerJob(table, reducer, job, partitioner, quorumAddress, serverClass, serverImpl,
true);
Job job, Class partitioner, String quorumAddress) throws IOException {
initTableReducerJob(table, reducer, job, partitioner, quorumAddress, true);
}

/**
Expand All @@ -597,16 +593,13 @@ public static void initTableReducerJob(String table, Class<? extends TableReduce
* <code> &lt;hbase.zookeeper.quorum&gt;:&lt;
* hbase.zookeeper.client.port&gt;:&lt;zookeeper.znode.parent&gt;
* </code> such as <code>server,server2,server3:2181:/hbase</code>.
* @param serverClass redefined hbase.regionserver.class
* @param serverImpl redefined hbase.regionserver.impl
* @param addDependencyJars upload HBase jars and jars for any of the configured job classes via
* the distributed cache (tmpjars).
* @throws IOException When determining the region count fails.
*/
public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl,
boolean addDependencyJars) throws IOException {

Job job, Class partitioner, String quorumAddress, boolean addDependencyJars)
throws IOException {
Configuration conf = job.getConfiguration();
HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
job.setOutputFormatClass(TableOutputFormat.class);
Expand All @@ -620,10 +613,6 @@ public static void initTableReducerJob(String table, Class<? extends TableReduce
ZKConfig.validateClusterKey(quorumAddress);
conf.set(TableOutputFormat.QUORUM_ADDRESS, quorumAddress);
}
if (serverClass != null && serverImpl != null) {
conf.set(TableOutputFormat.REGION_SERVER_CLASS, serverClass);
conf.set(TableOutputFormat.REGION_SERVER_IMPL, serverImpl);
}
job.setOutputKeyClass(ImmutableBytesWritable.class);
job.setOutputValueClass(Writable.class);
if (partitioner == HRegionPartitioner.class) {
Expand All @@ -643,6 +632,72 @@ public static void initTableReducerJob(String table, Class<? extends TableReduce
initCredentials(job);
}

/**
* Use this before submitting a TableReduce job. It will appropriately set up the JobConf.
* @param table The output table.
* @param reducer The reducer class to use.
* @param job The current job to adjust. Make sure the passed job is carrying all
* necessary HBase configuration.
* @param partitioner Partitioner to use. Pass <code>null</code> to use default partitioner.
* @param quorumAddress Distant cluster to write to; default is null for output to the cluster
* that is designated in <code>hbase-site.xml</code>. Set this String to the
* zookeeper ensemble of an alternate remote cluster when you would have the
* reduce write a cluster that is other than the default; e.g. copying tables
* between clusters, the source would be designated by
* <code>hbase-site.xml</code> and this param would have the ensemble address
* of the remote cluster. The format to pass is particular. Pass
* <code> &lt;hbase.zookeeper.quorum&gt;:&lt;
* hbase.zookeeper.client.port&gt;:&lt;zookeeper.znode.parent&gt;
* </code> such as <code>server,server2,server3:2181:/hbase</code>.
* @param serverClass redefined hbase.regionserver.class
* @param serverImpl redefined hbase.regionserver.impl
* @throws IOException When determining the region count fails.
* @deprecated Since 2.5.9, 2.6.1, 2.7.0, will be removed in 4.0.0. The {@code serverClass} and
* {@code serverImpl} do not take effect any more, just use
* {@link #initTableReducerJob(String, Class, Job, Class, String)} instead.
* @see #initTableReducerJob(String, Class, Job, Class, String)
*/
@Deprecated
public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl)
throws IOException {
initTableReducerJob(table, reducer, job, partitioner, quorumAddress);
}

/**
* Use this before submitting a TableReduce job. It will appropriately set up the JobConf.
* @param table The output table.
* @param reducer The reducer class to use.
* @param job The current job to adjust. Make sure the passed job is carrying all
* necessary HBase configuration.
* @param partitioner Partitioner to use. Pass <code>null</code> to use default partitioner.
* @param quorumAddress Distant cluster to write to; default is null for output to the cluster
* that is designated in <code>hbase-site.xml</code>. Set this String to
* the zookeeper ensemble of an alternate remote cluster when you would
* have the reduce write a cluster that is other than the default; e.g.
* copying tables between clusters, the source would be designated by
* <code>hbase-site.xml</code> and this param would have the ensemble
* address of the remote cluster. The format to pass is particular. Pass
* <code> &lt;hbase.zookeeper.quorum&gt;:&lt;
* hbase.zookeeper.client.port&gt;:&lt;zookeeper.znode.parent&gt;
* </code> such as <code>server,server2,server3:2181:/hbase</code>.
* @param serverClass redefined hbase.regionserver.class
* @param serverImpl redefined hbase.regionserver.impl
* @param addDependencyJars upload HBase jars and jars for any of the configured job classes via
* the distributed cache (tmpjars).
* @throws IOException When determining the region count fails.
* @deprecated Since 2.5.9, 2.6.1, 2.7.0, will be removed in 4.0.0. The {@code serverClass} and
* {@code serverImpl} do not take effect any more, just use
* {@link #initTableReducerJob(String, Class, Job, Class, String, boolean)} instead.
* @see #initTableReducerJob(String, Class, Job, Class, String, boolean)
*/
@Deprecated
public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl,
boolean addDependencyJars) throws IOException {
initTableReducerJob(table, reducer, job, partitioner, quorumAddress, addDependencyJars);
}

/**
* Ensures that the given number of reduce tasks for the given job configuration does not exceed
* the number of regions for the given table.
Expand Down
22 changes: 13 additions & 9 deletions hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,16 +66,26 @@ public class TableOutputFormat<KEY> extends OutputFormat<KEY, Mutation> implemen
* Optional job parameter to specify a peer cluster. Used specifying remote cluster when copying
* between hbase clusters (the source is picked up from <code>hbase-site.xml</code>).
* @see TableMapReduceUtil#initTableReducerJob(String, Class, org.apache.hadoop.mapreduce.Job,
* Class, String, String, String)
* Class, String)
*/
public static final String QUORUM_ADDRESS = OUTPUT_CONF_PREFIX + "quorum";

/** Optional job parameter to specify peer cluster's ZK client port */
public static final String QUORUM_PORT = OUTPUT_CONF_PREFIX + "quorum.port";

/** Optional specification of the rs class name of the peer cluster */
/**
* Optional specification of the rs class name of the peer cluster.
* @deprecated Since 2.5.9, 2.6.1 and 2.7.0, will be removed in 4.0.0. Does not take effect from
* long ago, see HBASE-6044.
*/
@Deprecated
public static final String REGION_SERVER_CLASS = OUTPUT_CONF_PREFIX + "rs.class";
/** Optional specification of the rs impl name of the peer cluster */
/**
* Optional specification of the rs impl name of the peer cluster
* @deprecated Since 2.5.9, 2.6.1 and 2.7.0, will be removed in 4.0.0. Does not take effect from
* long ago, see HBASE-6044.
*/
@Deprecated
public static final String REGION_SERVER_IMPL = OUTPUT_CONF_PREFIX + "rs.impl";

/** The configuration. */
Expand Down Expand Up @@ -208,15 +218,9 @@ public void setConf(Configuration otherConf) {

String address = otherConf.get(QUORUM_ADDRESS);
int zkClientPort = otherConf.getInt(QUORUM_PORT, 0);
String serverClass = otherConf.get(REGION_SERVER_CLASS);
String serverImpl = otherConf.get(REGION_SERVER_IMPL);

try {
this.conf = HBaseConfiguration.createClusterConf(otherConf, address, OUTPUT_CONF_PREFIX);

if (serverClass != null) {
this.conf.set(HConstants.REGION_SERVER_IMPL, serverImpl);
}
if (zkClientPort != 0) {
this.conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
}
Expand Down

0 comments on commit 771e1d9

Please sign in to comment.