Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HBASE-28616 Remove/Deprecated the rs.* related configuration in Table… #5946

Merged
merged 1 commit into from
May 29, 2024
+87 −30
Merged

HBASE-28616 Remove/Deprecated the rs.* related configuration in Table… #5946

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...rc/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ public int runCopier(String outputDir) throws Exception {
}
job.getConfiguration().setBoolean("mapreduce.map.speculative", false);
job.getConfiguration().setBoolean("mapreduce.reduce.speculative", false);
TableMapReduceUtil.initTableReducerJob(COMMON_TABLE_NAME, null, job, null, null, null, null);
TableMapReduceUtil.initTableReducerJob(COMMON_TABLE_NAME, null, job);
TableMapReduceUtil.addDependencyJars(job);
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
AbstractHBaseTool.class);
Expand Down
3 changes: 1 addition & 2 deletions hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,8 +181,7 @@ public Job createSubmittableJob(String[] args) throws IOException {
}
} else {
initCopyTableMapperReducerJob(job, scan);
TableMapReduceUtil.initTableReducerJob(dstTableName, null, job, null, peerAddress, null,
null);
TableMapReduceUtil.initTableReducerJob(dstTableName, null, job, null, peerAddress);
}

return job;
Expand Down
3 changes: 1 addition & 2 deletions hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,8 +167,7 @@ public Job createSubmittableJob(String[] args) throws IOException {
} else {
// No reducers. Just write straight to table. Call initTableReducerJob
// because it sets up the TableOutputFormat.
TableMapReduceUtil.initTableReducerJob(targetTableName, null, job, null, targetZkCluster,
null, null);
TableMapReduceUtil.initTableReducerJob(targetTableName, null, job, null, targetZkCluster);

// would be nice to add an option for bulk load instead
}
Expand Down
87 changes: 71 additions & 16 deletions hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -549,7 +549,7 @@ public static void initTableReducerJob(String table, Class<? extends TableReduce
*/
public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
Job job, Class partitioner) throws IOException {
initTableReducerJob(table, reducer, job, partitioner, null, null, null);
initTableReducerJob(table, reducer, job, partitioner, null);
}

/**
Expand All @@ -569,15 +569,11 @@ public static void initTableReducerJob(String table, Class<? extends TableReduce
* <code> &lt;hbase.zookeeper.quorum&gt;:&lt;
* hbase.zookeeper.client.port&gt;:&lt;zookeeper.znode.parent&gt;
* </code> such as <code>server,server2,server3:2181:/hbase</code>.
* @param serverClass redefined hbase.regionserver.class
* @param serverImpl redefined hbase.regionserver.impl
* @throws IOException When determining the region count fails.
*/
public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl)
throws IOException {
initTableReducerJob(table, reducer, job, partitioner, quorumAddress, serverClass, serverImpl,
true);
Job job, Class partitioner, String quorumAddress) throws IOException {
initTableReducerJob(table, reducer, job, partitioner, quorumAddress, true);
}

/**
Expand All @@ -597,16 +593,13 @@ public static void initTableReducerJob(String table, Class<? extends TableReduce
* <code> &lt;hbase.zookeeper.quorum&gt;:&lt;
* hbase.zookeeper.client.port&gt;:&lt;zookeeper.znode.parent&gt;
* </code> such as <code>server,server2,server3:2181:/hbase</code>.
* @param serverClass redefined hbase.regionserver.class
* @param serverImpl redefined hbase.regionserver.impl
* @param addDependencyJars upload HBase jars and jars for any of the configured job classes via
* the distributed cache (tmpjars).
* @throws IOException When determining the region count fails.
*/
public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl,
boolean addDependencyJars) throws IOException {

Job job, Class partitioner, String quorumAddress, boolean addDependencyJars)
throws IOException {
Configuration conf = job.getConfiguration();
HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
job.setOutputFormatClass(TableOutputFormat.class);
Expand All @@ -620,10 +613,6 @@ public static void initTableReducerJob(String table, Class<? extends TableReduce
ZKConfig.validateClusterKey(quorumAddress);
conf.set(TableOutputFormat.QUORUM_ADDRESS, quorumAddress);
}
if (serverClass != null && serverImpl != null) {
conf.set(TableOutputFormat.REGION_SERVER_CLASS, serverClass);
conf.set(TableOutputFormat.REGION_SERVER_IMPL, serverImpl);
}
job.setOutputKeyClass(ImmutableBytesWritable.class);
job.setOutputValueClass(Writable.class);
if (partitioner == HRegionPartitioner.class) {
Expand All @@ -643,6 +632,72 @@ public static void initTableReducerJob(String table, Class<? extends TableReduce
initCredentials(job);
}

/**
* Use this before submitting a TableReduce job. It will appropriately set up the JobConf.
* @param table The output table.
* @param reducer The reducer class to use.
* @param job The current job to adjust. Make sure the passed job is carrying all
* necessary HBase configuration.
* @param partitioner Partitioner to use. Pass <code>null</code> to use default partitioner.
* @param quorumAddress Distant cluster to write to; default is null for output to the cluster
* that is designated in <code>hbase-site.xml</code>. Set this String to the
* zookeeper ensemble of an alternate remote cluster when you would have the
* reduce write a cluster that is other than the default; e.g. copying tables
* between clusters, the source would be designated by
* <code>hbase-site.xml</code> and this param would have the ensemble address
* of the remote cluster. The format to pass is particular. Pass
* <code> &lt;hbase.zookeeper.quorum&gt;:&lt;
* hbase.zookeeper.client.port&gt;:&lt;zookeeper.znode.parent&gt;
* </code> such as <code>server,server2,server3:2181:/hbase</code>.
* @param serverClass redefined hbase.regionserver.class
* @param serverImpl redefined hbase.regionserver.impl
* @throws IOException When determining the region count fails.
* @deprecated Since 2.5.9, 2.6.1, 2.7.0, will be removed in 4.0.0. The {@code serverClass} and
* {@code serverImpl} do not take effect any more, just use
* {@link #initTableReducerJob(String, Class, Job, Class, String)} instead.
* @see #initTableReducerJob(String, Class, Job, Class, String)
*/
@Deprecated
public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl)
throws IOException {
initTableReducerJob(table, reducer, job, partitioner, quorumAddress);
}

/**
* Use this before submitting a TableReduce job. It will appropriately set up the JobConf.
* @param table The output table.
* @param reducer The reducer class to use.
* @param job The current job to adjust. Make sure the passed job is carrying all
* necessary HBase configuration.
* @param partitioner Partitioner to use. Pass <code>null</code> to use default partitioner.
* @param quorumAddress Distant cluster to write to; default is null for output to the cluster
* that is designated in <code>hbase-site.xml</code>. Set this String to
* the zookeeper ensemble of an alternate remote cluster when you would
* have the reduce write a cluster that is other than the default; e.g.
* copying tables between clusters, the source would be designated by
* <code>hbase-site.xml</code> and this param would have the ensemble
* address of the remote cluster. The format to pass is particular. Pass
* <code> &lt;hbase.zookeeper.quorum&gt;:&lt;
* hbase.zookeeper.client.port&gt;:&lt;zookeeper.znode.parent&gt;
* </code> such as <code>server,server2,server3:2181:/hbase</code>.
* @param serverClass redefined hbase.regionserver.class
* @param serverImpl redefined hbase.regionserver.impl
* @param addDependencyJars upload HBase jars and jars for any of the configured job classes via
* the distributed cache (tmpjars).
* @throws IOException When determining the region count fails.
* @deprecated Since 2.5.9, 2.6.1, 2.7.0, will be removed in 4.0.0. The {@code serverClass} and
* {@code serverImpl} do not take effect any more, just use
* {@link #initTableReducerJob(String, Class, Job, Class, String, boolean)} instead.
* @see #initTableReducerJob(String, Class, Job, Class, String, boolean)
*/
@Deprecated
public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl,
boolean addDependencyJars) throws IOException {
initTableReducerJob(table, reducer, job, partitioner, quorumAddress, addDependencyJars);
}

/**
* Ensures that the given number of reduce tasks for the given job configuration does not exceed
* the number of regions for the given table.
Expand Down
22 changes: 13 additions & 9 deletions hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,16 +66,26 @@ public class TableOutputFormat<KEY> extends OutputFormat<KEY, Mutation> implemen
* Optional job parameter to specify a peer cluster. Used specifying remote cluster when copying
* between hbase clusters (the source is picked up from <code>hbase-site.xml</code>).
* @see TableMapReduceUtil#initTableReducerJob(String, Class, org.apache.hadoop.mapreduce.Job,
* Class, String, String, String)
* Class, String)
*/
public static final String QUORUM_ADDRESS = OUTPUT_CONF_PREFIX + "quorum";

/** Optional job parameter to specify peer cluster's ZK client port */
public static final String QUORUM_PORT = OUTPUT_CONF_PREFIX + "quorum.port";

/** Optional specification of the rs class name of the peer cluster */
/**
* Optional specification of the rs class name of the peer cluster.
* @deprecated Since 2.5.9, 2.6.1 and 2.7.0, will be removed in 4.0.0. Does not take effect from
* long ago, see HBASE-6044.
*/
@Deprecated
public static final String REGION_SERVER_CLASS = OUTPUT_CONF_PREFIX + "rs.class";
/** Optional specification of the rs impl name of the peer cluster */
/**
* Optional specification of the rs impl name of the peer cluster
* @deprecated Since 2.5.9, 2.6.1 and 2.7.0, will be removed in 4.0.0. Does not take effect from
* long ago, see HBASE-6044.
*/
@Deprecated
public static final String REGION_SERVER_IMPL = OUTPUT_CONF_PREFIX + "rs.impl";

/** The configuration. */
Expand Down Expand Up @@ -208,15 +218,9 @@ public void setConf(Configuration otherConf) {

String address = otherConf.get(QUORUM_ADDRESS);
int zkClientPort = otherConf.getInt(QUORUM_PORT, 0);
String serverClass = otherConf.get(REGION_SERVER_CLASS);
String serverImpl = otherConf.get(REGION_SERVER_IMPL);

try {
this.conf = HBaseConfiguration.createClusterConf(otherConf, address, OUTPUT_CONF_PREFIX);

if (serverClass != null) {
this.conf.set(HConstants.REGION_SERVER_IMPL, serverImpl);
}
if (zkClientPort != 0) {
this.conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
}
Expand Down