Skip to content

Commit 1194ec3

Browse files
anuengineeranuengineer
committed
HDDS-17. Add node to container map class to simplify state in SCM.
Contributed by Anu Engineer.
1 parent 4132855 commit 1194ec3

File tree

9 files changed

+631
-4
lines changed

9 files changed

+631
-4
lines changed

‎hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerID.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ public class ContainerID implements Comparable {
3838
*/
3939
public ContainerID(long id) {
4040
Preconditions.checkState(id > 0,
41-
"Container ID should be a positive int");
41+
"Container ID should be a positive long. "+ id);
4242
this.id = id;
4343
}
4444

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/package-info.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,6 @@
1717
*/
1818

1919
/**
20-
* Container States management package.
20+
* Container States package.
2121
*/
2222
package org.apache.hadoop.hdds.scm.container.states;

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ public enum ResultCodes {
114114
FAILED_TO_FIND_BLOCK,
115115
IO_EXCEPTION,
116116
UNEXPECTED_CONTAINER_STATE,
117-
SCM_NOT_INITIALIZED
117+
SCM_NOT_INITIALIZED,
118+
DUPLICATE_DATANODE,
119+
NO_SUCH_DATANODE
118120
}
119121
}

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with this
4+
* work for additional information regarding copyright ownership. The ASF
5+
* licenses this file to you under the Apache License, Version 2.0 (the
6+
* "License"); you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
* License for the specific language governing permissions and limitations under
15+
* the License.
16+
*
17+
*/
18+
19+
package org.apache.hadoop.hdds.scm.node.states;
20+
21+
import com.google.common.base.Preconditions;
22+
import org.apache.hadoop.hdds.scm.container.ContainerID;
23+
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
24+
25+
import java.util.Collections;
26+
import java.util.Map;
27+
import java.util.Set;
28+
import java.util.TreeSet;
29+
import java.util.UUID;
30+
import java.util.concurrent.ConcurrentHashMap;
31+
32+
import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.DUPLICATE_DATANODE;
33+
import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.NO_SUCH_DATANODE;
34+
35+
/**
36+
* This data structure maintains the list of containers that is on a datanode.
37+
* This information is built from the DN container reports.
38+
*/
39+
public class Node2ContainerMap {
40+
private final Map<UUID, Set<ContainerID>> dn2ContainerMap;
41+
42+
/**
43+
* Constructs a Node2ContainerMap Object.
44+
*/
45+
public Node2ContainerMap() {
46+
dn2ContainerMap = new ConcurrentHashMap<>();
47+
}
48+
49+
/**
50+
* Returns true if this a datanode that is already tracked by
51+
* Node2ContainerMap.
52+
*
53+
* @param datanodeID - UUID of the Datanode.
54+
* @return True if this is tracked, false if this map does not know about it.
55+
*/
56+
public boolean isKnownDatanode(UUID datanodeID) {
57+
Preconditions.checkNotNull(datanodeID);
58+
return dn2ContainerMap.containsKey(datanodeID);
59+
}
60+
61+
/**
62+
* Insert a new datanode into Node2Container Map.
63+
*
64+
* @param datanodeID -- Datanode UUID
65+
* @param containerIDs - List of ContainerIDs.
66+
*/
67+
public void insertNewDatanode(UUID datanodeID, Set<ContainerID> containerIDs)
68+
throws SCMException {
69+
Preconditions.checkNotNull(containerIDs);
70+
Preconditions.checkNotNull(datanodeID);
71+
if(dn2ContainerMap.putIfAbsent(datanodeID, containerIDs) != null) {
72+
throw new SCMException("Node already exists in the map",
73+
DUPLICATE_DATANODE);
74+
}
75+
}
76+
77+
/**
78+
* Updates the Container list of an existing DN.
79+
*
80+
* @param datanodeID - UUID of DN.
81+
* @param containers - Set of Containers tht is present on DN.
82+
* @throws SCMException - if we don't know about this datanode, for new DN
83+
* use insertNewDatanode.
84+
*/
85+
public void updateDatanodeMap(UUID datanodeID, Set<ContainerID> containers)
86+
throws SCMException {
87+
Preconditions.checkNotNull(datanodeID);
88+
Preconditions.checkNotNull(containers);
89+
if(dn2ContainerMap.computeIfPresent(datanodeID, (k, v) -> v) == null){
90+
throw new SCMException("No such datanode", NO_SUCH_DATANODE);
91+
}
92+
}
93+
94+
/**
95+
* Removes datanode Entry from the map
96+
* @param datanodeID - Datanode ID.
97+
*/
98+
public void removeDatanode(UUID datanodeID) {
99+
Preconditions.checkNotNull(datanodeID);
100+
dn2ContainerMap.computeIfPresent(datanodeID, (k, v) -> null);
101+
}
102+
103+
/**
104+
* Returns null if there no containers associated with this datanode ID.
105+
*
106+
* @param datanode - UUID
107+
* @return Set of containers or Null.
108+
*/
109+
public Set<ContainerID> getContainers(UUID datanode) {
110+
Preconditions.checkNotNull(datanode);
111+
return dn2ContainerMap.computeIfPresent(datanode, (k, v) ->
112+
Collections.unmodifiableSet(v));
113+
}
114+
115+
public ReportResult processReport(UUID datanodeID, Set<ContainerID>
116+
containers) {
117+
Preconditions.checkNotNull(datanodeID);
118+
Preconditions.checkNotNull(containers);
119+
120+
if (!isKnownDatanode(datanodeID)) {
121+
return ReportResult.ReportResultBuilder.newBuilder()
122+
.setStatus(ReportStatus.NEW_DATANODE_FOUND)
123+
.setNewContainers(containers)
124+
.build();
125+
}
126+
127+
// Conditions like Zero length containers should be handled by removeAll.
128+
Set<ContainerID> currentSet = dn2ContainerMap.get(datanodeID);
129+
TreeSet<ContainerID> newContainers = new TreeSet<>(containers);
130+
newContainers.removeAll(currentSet);
131+
132+
TreeSet<ContainerID> missingContainers = new TreeSet<>(currentSet);
133+
missingContainers.removeAll(containers);
134+
135+
if (newContainers.isEmpty() && missingContainers.isEmpty()) {
136+
return ReportResult.ReportResultBuilder.newBuilder()
137+
.setStatus(ReportStatus.ALL_IS_WELL)
138+
.build();
139+
}
140+
141+
if (newContainers.isEmpty() && !missingContainers.isEmpty()) {
142+
return ReportResult.ReportResultBuilder.newBuilder()
143+
.setStatus(ReportStatus.MISSING_CONTAINERS)
144+
.setMissingContainers(missingContainers)
145+
.build();
146+
}
147+
148+
if (!newContainers.isEmpty() && missingContainers.isEmpty()) {
149+
return ReportResult.ReportResultBuilder.newBuilder()
150+
.setStatus(ReportStatus.NEW_CONTAINERS_FOUND)
151+
.setNewContainers(newContainers)
152+
.build();
153+
}
154+
155+
if (!newContainers.isEmpty() && !missingContainers.isEmpty()) {
156+
return ReportResult.ReportResultBuilder.newBuilder()
157+
.setStatus(ReportStatus.MISSING_AND_NEW_CONTAINERS_FOUND)
158+
.setNewContainers(newContainers)
159+
.setMissingContainers(missingContainers)
160+
.build();
161+
}
162+
163+
// default status & Make compiler happy
164+
return ReportResult.ReportResultBuilder.newBuilder()
165+
.setStatus(ReportStatus.ALL_IS_WELL)
166+
.build();
167+
}
168+
169+
170+
171+
172+
173+
/**
174+
* Results possible from processing a container report by
175+
* Node2ContainerMapper.
176+
*/
177+
public enum ReportStatus {
178+
ALL_IS_WELL,
179+
MISSING_CONTAINERS,
180+
NEW_CONTAINERS_FOUND,
181+
MISSING_AND_NEW_CONTAINERS_FOUND,
182+
NEW_DATANODE_FOUND
183+
}
184+
}

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/ReportResult.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*
18+
*/
19+
20+
package org.apache.hadoop.hdds.scm.node.states;
21+
22+
import org.apache.hadoop.hdds.scm.container.ContainerID;
23+
24+
import java.util.Set;
25+
26+
/**
27+
* A Container Report gets processsed by the Node2Container and returns the
28+
* Report Result class.
29+
*/
30+
public class ReportResult {
31+
private Node2ContainerMap.ReportStatus status;
32+
private Set<ContainerID> missingContainers;
33+
private Set<ContainerID> newContainers;
34+
35+
ReportResult(Node2ContainerMap.ReportStatus status,
36+
Set<ContainerID> missingContainers,
37+
Set<ContainerID> newContainers) {
38+
this.status = status;
39+
this.missingContainers = missingContainers;
40+
this.newContainers = newContainers;
41+
}
42+
43+
public Node2ContainerMap.ReportStatus getStatus() {
44+
return status;
45+
}
46+
47+
public Set<ContainerID> getMissingContainers() {
48+
return missingContainers;
49+
}
50+
51+
public Set<ContainerID> getNewContainers() {
52+
return newContainers;
53+
}
54+
55+
static class ReportResultBuilder {
56+
private Node2ContainerMap.ReportStatus status;
57+
private Set<ContainerID> missingContainers;
58+
private Set<ContainerID> newContainers;
59+
60+
static ReportResultBuilder newBuilder() {
61+
return new ReportResultBuilder();
62+
}
63+
64+
public ReportResultBuilder setStatus(
65+
Node2ContainerMap.ReportStatus newstatus) {
66+
this.status = newstatus;
67+
return this;
68+
}
69+
70+
public ReportResultBuilder setMissingContainers(
71+
Set<ContainerID> missingContainersLit) {
72+
this.missingContainers = missingContainersLit;
73+
return this;
74+
}
75+
76+
public ReportResultBuilder setNewContainers(
77+
Set<ContainerID> newContainersList) {
78+
this.newContainers = newContainersList;
79+
return this;
80+
}
81+
82+
ReportResult build() {
83+
return new ReportResult(status, missingContainers, newContainers);
84+
}
85+
}
86+
}

‎hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/package-info.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with this
4+
* work for additional information regarding copyright ownership. The ASF
5+
* licenses this file to you under the Apache License, Version 2.0 (the
6+
* "License"); you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
* License for the specific language governing permissions and limitations under
15+
* the License.
16+
*
17+
*/
18+
19+
/**
20+
* Node States package.
21+
*/
22+
package org.apache.hadoop.hdds.scm.node.states;

0 commit comments

Comments
 (0)