Skip to content

Commit

Permalink
HDDS-815. Rename HDDS config keys prefixed with dfs. (apache#6274)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sarvekshayr
sarvekshayr authored Feb 27, 2024
1 parent 083a45e commit 7939faf
Show file tree
Hide file tree
Showing 14 changed files with 144 additions and 77 deletions.
63 changes: 62 additions & 1 deletion hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;

import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.ratis.server.RaftServerConfigKeys;

import static java.util.Collections.unmodifiableSortedSet;
Expand Down Expand Up @@ -323,7 +324,67 @@ private static void addDeprecatedKeys() {
new DeprecationDelta("ozone.scm.chunk.layout",
ScmConfigKeys.OZONE_SCM_CONTAINER_LAYOUT_KEY),
new DeprecationDelta("hdds.datanode.replication.work.dir",
OZONE_CONTAINER_COPY_WORKDIR)
OZONE_CONTAINER_COPY_WORKDIR),
new DeprecationDelta("dfs.container.chunk.write.sync",
OzoneConfigKeys.DFS_CONTAINER_CHUNK_WRITE_SYNC_KEY),
new DeprecationDelta("dfs.container.ipc",
OzoneConfigKeys.DFS_CONTAINER_IPC_PORT),
new DeprecationDelta("dfs.container.ipc.random.port",
OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT),
new DeprecationDelta("dfs.container.ratis.admin.port",
OzoneConfigKeys.DFS_CONTAINER_RATIS_ADMIN_PORT),
new DeprecationDelta("dfs.container.ratis.datanode.storage.dir",
OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR),
new DeprecationDelta("dfs.container.ratis.datastream.enabled",
OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED),
new DeprecationDelta("dfs.container.ratis.datastream.port",
OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_PORT),
new DeprecationDelta("dfs.container.ratis.datastream.random.port",
OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT),
new DeprecationDelta("dfs.container.ratis.enabled",
ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY),
new DeprecationDelta("dfs.container.ratis.ipc",
OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT),
new DeprecationDelta("dfs.container.ratis.ipc.random.port",
OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT),
new DeprecationDelta("dfs.container.ratis.leader.pending.bytes.limit",
ScmConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT),
new DeprecationDelta("dfs.container.ratis.log.appender.queue.byte-limit",
ScmConfigKeys.DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT),
new DeprecationDelta("dfs.container.ratis.log.appender.queue.num-elements",
ScmConfigKeys.DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS),
new DeprecationDelta("dfs.container.ratis.log.purge.gap",
ScmConfigKeys.DFS_CONTAINER_RATIS_LOG_PURGE_GAP),
new DeprecationDelta("dfs.container.ratis.log.queue.byte-limit",
ScmConfigKeys.DFS_CONTAINER_RATIS_LOG_QUEUE_BYTE_LIMIT),
new DeprecationDelta("dfs.container.ratis.log.queue.num-elements",
ScmConfigKeys.DFS_CONTAINER_RATIS_LOG_QUEUE_NUM_ELEMENTS),
new DeprecationDelta("dfs.container.ratis.num.container.op.executors",
ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY),
new DeprecationDelta("dfs.container.ratis.num.write.chunk.threads.per.volume",
ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_PER_VOLUME),
new DeprecationDelta("dfs.container.ratis.replication.level",
ScmConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY),
new DeprecationDelta("dfs.container.ratis.rpc.type",
ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY),
new DeprecationDelta("dfs.container.ratis.segment.preallocated.size",
ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY),
new DeprecationDelta("dfs.container.ratis.segment.size",
ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY),
new DeprecationDelta("dfs.container.ratis.server.port",
OzoneConfigKeys.DFS_CONTAINER_RATIS_SERVER_PORT),
new DeprecationDelta("dfs.container.ratis.statemachinedata.sync.retries",
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES),
new DeprecationDelta("dfs.container.ratis.statemachinedata.sync.timeout",
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT),
new DeprecationDelta("dfs.container.ratis.statemachine.max.pending.apply-transactions",
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINE_MAX_PENDING_APPLY_TXNS),
new DeprecationDelta("dfs.ratis.leader.election.minimum.timeout.duration",
ScmConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY),
new DeprecationDelta("dfs.ratis.server.retry-cache.timeout.duration",
ScmConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY),
new DeprecationDelta("dfs.ratis.snapshot.threshold",
ScmConfigKeys.DFS_RATIS_SNAPSHOT_THRESHOLD_KEY)
});
}

Expand Down
38 changes: 19 additions & 19 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,93 +42,93 @@ public final class ScmConfigKeys {
"ozone.scm.db.dirs.permissions";

public static final String DFS_CONTAINER_RATIS_ENABLED_KEY
= "dfs.container.ratis.enabled";
= "hdds.container.ratis.enabled";
public static final boolean DFS_CONTAINER_RATIS_ENABLED_DEFAULT
= false;
public static final String DFS_CONTAINER_RATIS_RPC_TYPE_KEY
= "dfs.container.ratis.rpc.type";
= "hdds.container.ratis.rpc.type";
public static final String DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT
= "GRPC";
public static final String
DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_PER_VOLUME
= "dfs.container.ratis.num.write.chunk.threads.per.volume";
= "hdds.container.ratis.num.write.chunk.threads.per.volume";
public static final int
DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_PER_VOLUME_DEFAULT
= 10;
public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
= "dfs.container.ratis.replication.level";
= "hdds.container.ratis.replication.level";
public static final ReplicationLevel
DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT = ReplicationLevel.MAJORITY;
public static final String DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY
= "dfs.container.ratis.num.container.op.executors";
= "hdds.container.ratis.num.container.op.executors";
public static final int DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT
= 10;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
"dfs.container.ratis.segment.size";
"hdds.container.ratis.segment.size";
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
"64MB";
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY =
"dfs.container.ratis.segment.preallocated.size";
"hdds.container.ratis.segment.preallocated.size";
public static final String
DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = "4MB";
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
"dfs.container.ratis.statemachinedata.sync.timeout";
"hdds.container.ratis.statemachinedata.sync.timeout";
public static final TimeDuration
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
TimeDuration.valueOf(10, TimeUnit.SECONDS);
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES =
"dfs.container.ratis.statemachinedata.sync.retries";
"hdds.container.ratis.statemachinedata.sync.retries";
public static final String
DFS_CONTAINER_RATIS_STATEMACHINE_MAX_PENDING_APPLY_TXNS =
"dfs.container.ratis.statemachine.max.pending.apply-transactions";
"hdds.container.ratis.statemachine.max.pending.apply-transactions";
// The default value of maximum number of pending state machine apply
// transactions is kept same as default snapshot threshold.
public static final int
DFS_CONTAINER_RATIS_STATEMACHINE_MAX_PENDING_APPLY_TXNS_DEFAULT =
100000;
public static final String DFS_CONTAINER_RATIS_LOG_QUEUE_NUM_ELEMENTS =
"dfs.container.ratis.log.queue.num-elements";
"hdds.container.ratis.log.queue.num-elements";
public static final int DFS_CONTAINER_RATIS_LOG_QUEUE_NUM_ELEMENTS_DEFAULT =
1024;
public static final String DFS_CONTAINER_RATIS_LOG_QUEUE_BYTE_LIMIT =
"dfs.container.ratis.log.queue.byte-limit";
"hdds.container.ratis.log.queue.byte-limit";
public static final String DFS_CONTAINER_RATIS_LOG_QUEUE_BYTE_LIMIT_DEFAULT =
"4GB";
public static final String
DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS =
"dfs.container.ratis.log.appender.queue.num-elements";
"hdds.container.ratis.log.appender.queue.num-elements";
public static final int
DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS_DEFAULT = 1;
public static final String DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT =
"dfs.container.ratis.log.appender.queue.byte-limit";
"hdds.container.ratis.log.appender.queue.byte-limit";
public static final String
DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT = "32MB";
public static final String DFS_CONTAINER_RATIS_LOG_PURGE_GAP =
"dfs.container.ratis.log.purge.gap";
"hdds.container.ratis.log.purge.gap";
// TODO: Set to 1024 once RATIS issue around purge is fixed.
public static final int DFS_CONTAINER_RATIS_LOG_PURGE_GAP_DEFAULT =
1000000;
public static final String DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT =
"dfs.container.ratis.leader.pending.bytes.limit";
"hdds.container.ratis.leader.pending.bytes.limit";
public static final String
DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT = "1GB";

public static final String DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY =
"dfs.ratis.server.retry-cache.timeout.duration";
"hdds.ratis.server.retry-cache.timeout.duration";
public static final TimeDuration
DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT =
TimeDuration.valueOf(600000, TimeUnit.MILLISECONDS);
public static final String
DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY =
"dfs.ratis.leader.election.minimum.timeout.duration";
"hdds.ratis.leader.election.minimum.timeout.duration";
public static final TimeDuration
DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT =
TimeDuration.valueOf(5, TimeUnit.SECONDS);

public static final String DFS_RATIS_SNAPSHOT_THRESHOLD_KEY =
"dfs.ratis.snapshot.threshold";
"hdds.ratis.snapshot.threshold";
public static final long DFS_RATIS_SNAPSHOT_THRESHOLD_DEFAULT = 100000;

// TODO : this is copied from OzoneConsts, may need to move to a better place
Expand Down
22 changes: 11 additions & 11 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
@InterfaceStability.Unstable
public final class OzoneConfigKeys {
public static final String DFS_CONTAINER_IPC_PORT =
"dfs.container.ipc";
"hdds.container.ipc.port";
public static final int DFS_CONTAINER_IPC_PORT_DEFAULT = 9859;

public static final String OZONE_METADATA_DIRS = "ozone.metadata.dirs";
Expand All @@ -60,47 +60,47 @@ public final class OzoneConfigKeys {
* as {@link #DFS_CONTAINER_IPC_PORT_DEFAULT}.
*/
public static final String DFS_CONTAINER_IPC_RANDOM_PORT =
"dfs.container.ipc.random.port";
"hdds.container.ipc.random.port";
public static final boolean DFS_CONTAINER_IPC_RANDOM_PORT_DEFAULT =
false;

public static final String DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT =
"dfs.container.ratis.datastream.random.port";
"hdds.container.ratis.datastream.random.port";
public static final boolean
DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT_DEFAULT =
false;

public static final String DFS_CONTAINER_CHUNK_WRITE_SYNC_KEY =
"dfs.container.chunk.write.sync";
"hdds.container.chunk.write.sync";
public static final boolean DFS_CONTAINER_CHUNK_WRITE_SYNC_DEFAULT = false;
/**
* Ratis Port where containers listen to.
*/
public static final String DFS_CONTAINER_RATIS_IPC_PORT =
"dfs.container.ratis.ipc";
"hdds.container.ratis.ipc.port";
public static final int DFS_CONTAINER_RATIS_IPC_PORT_DEFAULT = 9858;
/**
* Ratis Port where containers listen to admin requests.
*/
public static final String DFS_CONTAINER_RATIS_ADMIN_PORT =
"dfs.container.ratis.admin.port";
"hdds.container.ratis.admin.port";
public static final int DFS_CONTAINER_RATIS_ADMIN_PORT_DEFAULT = 9857;
/**
* Ratis Port where containers listen to server-to-server requests.
*/
public static final String DFS_CONTAINER_RATIS_SERVER_PORT =
"dfs.container.ratis.server.port";
"hdds.container.ratis.server.port";
public static final int DFS_CONTAINER_RATIS_SERVER_PORT_DEFAULT = 9856;

/**
* Ratis Port where containers listen to datastream requests.
*/
public static final String DFS_CONTAINER_RATIS_DATASTREAM_ENABLED
= "dfs.container.ratis.datastream.enabled";
= "hdds.container.ratis.datastream.enabled";
public static final boolean DFS_CONTAINER_RATIS_DATASTREAM_ENABLED_DEFAULT
= false;
public static final String DFS_CONTAINER_RATIS_DATASTREAM_PORT
= "dfs.container.ratis.datastream.port";
= "hdds.container.ratis.datastream.port";
public static final int DFS_CONTAINER_RATIS_DATASTREAM_PORT_DEFAULT
= 9855;

Expand Down Expand Up @@ -134,7 +134,7 @@ public final class OzoneConfigKeys {
* a mini cluster is able to launch multiple containers on a node.
*/
public static final String DFS_CONTAINER_RATIS_IPC_RANDOM_PORT =
"dfs.container.ratis.ipc.random.port";
"hdds.container.ratis.ipc.random.port";
public static final boolean DFS_CONTAINER_RATIS_IPC_RANDOM_PORT_DEFAULT =
false;

Expand Down Expand Up @@ -368,7 +368,7 @@ public final class OzoneConfigKeys {
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT;

public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
"dfs.container.ratis.datanode.storage.dir";
"hdds.container.ratis.datanode.storage.dir";

public static final String DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY =
ScmConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY;
Expand Down
Loading

0 comments on commit 7939faf

Please sign in to comment.