Skip to content

Commit f93d2ea

Browse files
jian-hejian-he
committed
YARN-2312. Deprecated old ContainerId#getId API and updated MapReduce to use ContainerId#getContainerId instead. Contributed by Tsuyoshi OZAWA
1 parent 025b6c1 commit f93d2ea

File tree

24 files changed

+123
-71
lines changed

24 files changed

+123
-71
lines changed

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ private static void setupLog4jProperties(Task task,
157157

158158
public static List<String> getVMCommand(
159159
InetSocketAddress taskAttemptListenerAddr, Task task,
160-
ID jvmID) {
160+
JVMId jvmID) {
161161

162162
TaskAttemptID attemptID = task.getTaskID();
163163
JobConf conf = task.conf;

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/WrappedJvmID.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@
2323
*/
2424
public class WrappedJvmID extends JVMId {
2525

26-
public WrappedJvmID(JobID jobID, boolean mapTask, int nextInt) {
27-
super(jobID, mapTask, nextInt);
26+
public WrappedJvmID(JobID jobID, boolean mapTask, long nextLong) {
27+
super(jobID, mapTask, nextLong);
2828
}
2929

3030
}

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,9 +82,9 @@ public static void main(String[] args) throws Throwable {
8282
final InetSocketAddress address =
8383
NetUtils.createSocketAddrForHost(host, port);
8484
final TaskAttemptID firstTaskid = TaskAttemptID.forName(args[2]);
85-
int jvmIdInt = Integer.parseInt(args[3]);
85+
long jvmIdLong = Long.parseLong(args[3]);
8686
JVMId jvmId = new JVMId(firstTaskid.getJobID(),
87-
firstTaskid.getTaskType() == TaskType.MAP, jvmIdInt);
87+
firstTaskid.getTaskType() == TaskType.MAP, jvmIdLong);
8888

8989
// initialize metrics
9090
DefaultMetricsSystem.initialize(

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1513,8 +1513,8 @@ public void transition(final TaskAttemptImpl taskAttempt,
15131513
taskAttempt.remoteTask = taskAttempt.createRemoteTask();
15141514
taskAttempt.jvmID =
15151515
new WrappedJvmID(taskAttempt.remoteTask.getTaskID().getJobID(),
1516-
taskAttempt.remoteTask.isMapTask(), taskAttempt.container.getId()
1517-
.getId());
1516+
taskAttempt.remoteTask.isMapTask(),
1517+
taskAttempt.container.getId().getContainerId());
15181518
taskAttempt.taskAttemptListener.registerPendingTask(
15191519
taskAttempt.remoteTask, taskAttempt.jvmID);
15201520

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ public void handle(ContainerAllocatorEvent event) {
150150
// Assign the same container ID as the AM
151151
ContainerId cID =
152152
ContainerId.newInstance(getContext().getApplicationAttemptId(),
153-
this.containerId.getId());
153+
this.containerId.getContainerId());
154154
Container container = recordFactory.newRecordInstance(Container.class);
155155
container.setId(cID);
156156
NodeId nodeId = NodeId.newInstance(this.nmHost, this.nmPort);

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -237,9 +237,10 @@ public AllocateResponse allocate(AllocateRequest request)
237237
getContext().getApplicationAttemptId(),
238238
request.getResponseId() + i);
239239
containers.add(Container.newInstance(containerId,
240-
NodeId.newInstance("host" + containerId.getId(), 2345),
241-
"host" + containerId.getId() + ":5678",
242-
req.getCapability(), req.getPriority(), null));
240+
NodeId.newInstance(
241+
"host" + containerId.getContainerId(), 2345),
242+
"host" + containerId.getContainerId() + ":5678",
243+
req.getCapability(), req.getPriority(), null));
243244
}
244245
}
245246

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java‎

Lines changed: 55 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -23,23 +23,25 @@
2323
import java.io.IOException;
2424
import java.text.NumberFormat;
2525

26-
class JVMId extends ID {
26+
class JVMId {
2727
boolean isMap;
28-
JobID jobId;
28+
final JobID jobId;
29+
private long jvmId;
2930
private static final String JVM = "jvm";
31+
private static final char SEPARATOR = '_';
3032
private static NumberFormat idFormat = NumberFormat.getInstance();
3133
static {
3234
idFormat.setGroupingUsed(false);
3335
idFormat.setMinimumIntegerDigits(6);
3436
}
3537

36-
public JVMId(JobID jobId, boolean isMap, int id) {
37-
super(id);
38+
public JVMId(JobID jobId, boolean isMap, long id) {
39+
this.jvmId = id;
3840
this.isMap = isMap;
3941
this.jobId = jobId;
4042
}
4143

42-
public JVMId (String jtIdentifier, int jobId, boolean isMap, int id) {
44+
public JVMId (String jtIdentifier, int jobId, boolean isMap, long id) {
4345
this(new JobID(jtIdentifier, jobId), isMap, id);
4446
}
4547

@@ -53,27 +55,50 @@ public boolean isMapJVM() {
5355
public JobID getJobId() {
5456
return jobId;
5557
}
58+
59+
@Override
5660
public boolean equals(Object o) {
57-
if(o == null)
61+
// Generated by IntelliJ IDEA 13.1.
62+
if (this == o) {
63+
return true;
64+
}
65+
if (o == null || getClass() != o.getClass()) {
66+
return false;
67+
}
68+
69+
JVMId jvmId1 = (JVMId) o;
70+
71+
if (isMap != jvmId1.isMap) {
5872
return false;
59-
if(o.getClass().equals(this.getClass())) {
60-
JVMId that = (JVMId)o;
61-
return this.id==that.id
62-
&& this.isMap == that.isMap
63-
&& this.jobId.equals(that.jobId);
6473
}
65-
else return false;
74+
if (jvmId != jvmId1.jvmId) {
75+
return false;
76+
}
77+
if (!jobId.equals(jvmId1.jobId)) {
78+
return false;
79+
}
80+
81+
return true;
6682
}
6783

68-
/**Compare TaskInProgressIds by first jobIds, then by tip numbers. Reduces are
69-
* defined as greater then maps.*/
7084
@Override
71-
public int compareTo(org.apache.hadoop.mapreduce.ID o) {
72-
JVMId that = (JVMId)o;
85+
public int hashCode() {
86+
// Generated by IntelliJ IDEA 13.1.
87+
int result = (isMap ? 1 : 0);
88+
result = 31 * result + jobId.hashCode();
89+
result = 31 * result + (int) (jvmId ^ (jvmId >>> 32));
90+
return result;
91+
}
92+
93+
/**
94+
* Compare TaskInProgressIds by first jobIds, then by tip numbers. Reduces are
95+
* defined as greater then maps.
96+
**/
97+
public int compareTo(JVMId that) {
7398
int jobComp = this.jobId.compareTo(that.jobId);
7499
if(jobComp == 0) {
75100
if(this.isMap == that.isMap) {
76-
return this.id - that.id;
101+
return Long.valueOf(this.jvmId).compareTo(that.jvmId);
77102
} else {
78103
return this.isMap ? -1 : 1;
79104
}
@@ -87,6 +112,15 @@ public String toString() {
87112
return appendTo(new StringBuilder(JVM)).toString();
88113
}
89114

115+
/**
116+
* This method does NOT override org.apache.hadoop.mapred.ID to accept 64-bit
117+
* ID to support work-preserving RM restart.
118+
* @return 64-bit JVM id.
119+
*/
120+
public long getId() {
121+
return jvmId;
122+
}
123+
90124
/**
91125
* Add the unique id to the given StringBuilder.
92126
* @param builder the builder to append to
@@ -97,24 +131,17 @@ protected StringBuilder appendTo(StringBuilder builder) {
97131
append(SEPARATOR).
98132
append(isMap ? 'm' : 'r').
99133
append(SEPARATOR).
100-
append(idFormat.format(id));
101-
}
102-
103-
@Override
104-
public int hashCode() {
105-
return jobId.hashCode() * 11 + id;
134+
append(idFormat.format(jvmId));
106135
}
107-
108-
@Override
136+
109137
public void readFields(DataInput in) throws IOException {
110-
super.readFields(in);
138+
this.jvmId = in.readLong();
111139
this.jobId.readFields(in);
112140
this.isMap = in.readBoolean();
113141
}
114142

115-
@Override
116143
public void write(DataOutput out) throws IOException {
117-
super.write(out);
144+
out.writeLong(jvmId);
118145
jobId.write(out);
119146
out.writeBoolean(isMap);
120147
}

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -671,7 +671,8 @@ public void testContainerRollingLog() throws IOException,
671671
if (!foundAppMaster) {
672672
final ContainerId cid = ConverterUtils.toContainerId(
673673
containerPathComponent.getName());
674-
foundAppMaster = (cid.getId() == 1);
674+
foundAppMaster =
675+
((cid.getContainerId() & ContainerId.CONTAINER_ID_BITMASK)== 1);
675676
}
676677

677678
final FileStatus[] sysSiblings = localFs.globStatus(new Path(

‎hadoop-yarn-project/CHANGES.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,9 @@ Release 2.6.0 - UNRELEASED
318318
YARN-2583. Modified AggregatedLogDeletionService to be able to delete rolling
319319
aggregated logs. (Xuan Gong via zjshen)
320320

321+
YARN-2312. Deprecated old ContainerId#getId API and updated MapReduce to
322+
use ContainerId#getContainerId instead. (Tsuyoshi OZAWA via jianhe)
323+
321324
OPTIMIZATIONS
322325

323326
BUG FIXES

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
@Public
3636
@Stable
3737
public abstract class ContainerId implements Comparable<ContainerId>{
38+
public static final long CONTAINER_ID_BITMASK = 0xffffffffffL;
3839
private static final Splitter _SPLITTER = Splitter.on('_').trimResults();
3940
private static final String CONTAINER_PREFIX = "container";
4041
private static final String EPOCH_PREFIX = "e";
@@ -81,6 +82,7 @@ public static ContainerId newInstance(ApplicationAttemptId appAttemptId,
8182
* @return lower 32 bits of identifier of the <code>ContainerId</code>
8283
*/
8384
@Public
85+
@Deprecated
8486
@Stable
8587
public abstract int getId();
8688

@@ -184,7 +186,8 @@ public String toString() {
184186
sb.append(
185187
appAttemptIdAndEpochFormat.get().format(
186188
getApplicationAttemptId().getAttemptId())).append("_");
187-
sb.append(containerIdFormat.get().format(0xffffffffffL & getContainerId()));
189+
sb.append(containerIdFormat.get()
190+
.format(CONTAINER_ID_BITMASK & getContainerId()));
188191
return sb.toString();
189192
}
190193

0 commit comments

Comments
 (0)