Skip to content

Commit a43f444

Browse files
xiaoyuyaoxiaoyuyao
authored
HDDS-1543. Implement addAcl,removeAcl,setAcl,getAcl for Prefix. Contr… (#927)
1 parent 205dd2d commit a43f444

File tree

14 files changed

+786
-78
lines changed

14 files changed

+786
-78
lines changed

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@
4646
* <tr>
4747
* <td> 2 </td> <td> Bucket Lock </td>
4848
* </tr>
49+
* <tr>
50+
* <td> 3 </td> <td> Prefix Lock </td>
51+
* </tr>
4952
* </table>
5053
*
5154
* One cannot obtain a lower weight lock while holding a lock with higher
@@ -66,6 +69,7 @@ public final class OzoneManagerLock {
6669

6770
private static final String VOLUME_LOCK = "volumeLock";
6871
private static final String BUCKET_LOCK = "bucketLock";
72+
private static final String PREFIX_LOCK = "prefixLock";
6973
private static final String S3_BUCKET_LOCK = "s3BucketLock";
7074
private static final String S3_SECRET_LOCK = "s3SecretetLock";
7175

@@ -77,6 +81,7 @@ public final class OzoneManagerLock {
7781
() -> ImmutableMap.of(
7882
VOLUME_LOCK, new AtomicInteger(0),
7983
BUCKET_LOCK, new AtomicInteger(0),
84+
PREFIX_LOCK, new AtomicInteger(0),
8085
S3_BUCKET_LOCK, new AtomicInteger(0),
8186
S3_SECRET_LOCK, new AtomicInteger(0)
8287
)
@@ -241,4 +246,24 @@ public void releaseS3SecretLock(String awsAccessId) {
241246
manager.unlock(awsAccessId);
242247
myLocks.get().get(S3_SECRET_LOCK).decrementAndGet();
243248
}
249+
250+
public void acquirePrefixLock(String prefixPath) {
251+
if (hasAnyPrefixLock()) {
252+
throw new RuntimeException(
253+
"Thread '" + Thread.currentThread().getName() +
254+
"' cannot acquire prefix path lock while holding prefix " +
255+
"path lock(s) for path: " + prefixPath + ".");
256+
}
257+
manager.lock(prefixPath);
258+
myLocks.get().get(PREFIX_LOCK).incrementAndGet();
259+
}
260+
261+
private boolean hasAnyPrefixLock() {
262+
return myLocks.get().get(PREFIX_LOCK).get() != 0;
263+
}
264+
265+
public void releasePrefixLock(String prefixPath) {
266+
manager.unlock(prefixPath);
267+
myLocks.get().get(PREFIX_LOCK).decrementAndGet();
268+
}
244269
}

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,9 @@ public enum ResultCodes {
199199

200200
PERMISSION_DENIED, // Error codes used during acl validation
201201

202-
TIMEOUT // Error codes used during acl validation
202+
TIMEOUT, // Error codes used during acl validation
203+
204+
PREFIX_NOT_FOUND,
205+
203206
}
204207
}

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObj.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,19 @@ public StoreType getStoreType() {
7171

7272
public abstract String getKeyName();
7373

74+
/**
75+
* Get PrefixName.
76+
* A prefix name is like a key name under the bucket but
77+
* are mainly used for ACL for now and persisted into a separate prefix table.
78+
*
79+
* @return prefix name.
80+
*/
81+
public abstract String getPrefixName();
82+
83+
/**
84+
* Get full path of a key or prefix including volume and bucket.
85+
* @return full path of a key or prefix.
86+
*/
7487
public abstract String getPath();
7588

7689
/**
@@ -79,7 +92,8 @@ public StoreType getStoreType() {
7992
public enum ResourceType {
8093
VOLUME(OzoneConsts.VOLUME),
8194
BUCKET(OzoneConsts.BUCKET),
82-
KEY(OzoneConsts.KEY);
95+
KEY(OzoneConsts.KEY),
96+
PREFIX(OzoneConsts.PREFIX);
8397

8498
/**
8599
* String value for this Enum.

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneObjInfo.java‎

Lines changed: 51 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -23,32 +23,51 @@
2323

2424
/**
2525
* Class representing an ozone object.
26+
* It can be a volume with non-null volumeName (bucketName=null & name=null)
27+
* or a bucket with non-null volumeName and bucketName (name=null)
28+
* or a key with non-null volumeName, bucketName and key name
29+
* (via getKeyName)
30+
* or a prefix with non-null volumeName, bucketName and prefix name
31+
* (via getPrefixName)
2632
*/
2733
public final class OzoneObjInfo extends OzoneObj {
2834

2935
private final String volumeName;
3036
private final String bucketName;
31-
private final String keyName;
32-
37+
private final String name;
3338

39+
/**
40+
*
41+
* @param resType
42+
* @param storeType
43+
* @param volumeName
44+
* @param bucketName
45+
* @param name - keyName/PrefixName
46+
*/
3447
private OzoneObjInfo(ResourceType resType, StoreType storeType,
35-
String volumeName, String bucketName, String keyName) {
48+
String volumeName, String bucketName, String name) {
3649
super(resType, storeType);
3750
this.volumeName = volumeName;
3851
this.bucketName = bucketName;
39-
this.keyName = keyName;
52+
this.name = name;
4053
}
4154

4255
@Override
4356
public String getPath() {
4457
switch (getResourceType()) {
4558
case VOLUME:
46-
return getVolumeName();
59+
return OZONE_URI_DELIMITER + getVolumeName();
4760
case BUCKET:
48-
return getVolumeName() + OZONE_URI_DELIMITER + getBucketName();
61+
return OZONE_URI_DELIMITER + getVolumeName()
62+
+ OZONE_URI_DELIMITER + getBucketName();
4963
case KEY:
50-
return getVolumeName() + OZONE_URI_DELIMITER + getBucketName()
64+
return OZONE_URI_DELIMITER + getVolumeName()
65+
+ OZONE_URI_DELIMITER + getBucketName()
5166
+ OZONE_URI_DELIMITER + getKeyName();
67+
case PREFIX:
68+
return OZONE_URI_DELIMITER + getVolumeName()
69+
+ OZONE_URI_DELIMITER + getBucketName()
70+
+ OZONE_URI_DELIMITER + getPrefixName();
5271
default:
5372
throw new IllegalArgumentException("Unknown resource " +
5473
"type" + getResourceType());
@@ -67,9 +86,15 @@ public String getBucketName() {
6786

6887
@Override
6988
public String getKeyName() {
70-
return keyName;
89+
return name;
7190
}
7291

92+
@Override
93+
public String getPrefixName() {
94+
return name;
95+
}
96+
97+
7398
public static OzoneObjInfo fromProtobuf(OzoneManagerProtocolProtos.OzoneObj
7499
proto) {
75100
Builder builder = new Builder()
@@ -88,7 +113,7 @@ public static OzoneObjInfo fromProtobuf(OzoneManagerProtocolProtos.OzoneObj
88113
case BUCKET:
89114
if (tokens.length < 2) {
90115
throw new IllegalArgumentException("Unexpected argument for " +
91-
"Ozone key. Path:" + proto.getPath());
116+
"Ozone bucket. Path:" + proto.getPath());
92117
}
93118
builder.setVolumeName(tokens[0]);
94119
builder.setBucketName(tokens[1]);
@@ -102,6 +127,15 @@ public static OzoneObjInfo fromProtobuf(OzoneManagerProtocolProtos.OzoneObj
102127
builder.setBucketName(tokens[1]);
103128
builder.setKeyName(tokens[2]);
104129
break;
130+
case PREFIX:
131+
if (tokens.length < 3) {
132+
throw new IllegalArgumentException("Unexpected argument for " +
133+
"Ozone Prefix. Path:" + proto.getPath());
134+
}
135+
builder.setVolumeName(tokens[0]);
136+
builder.setBucketName(tokens[1]);
137+
builder.setPrefixName(tokens[2]);
138+
break;
105139
default:
106140
throw new IllegalArgumentException("Unexpected type for " +
107141
"Ozone key. Type:" + proto.getResType());
@@ -118,7 +152,7 @@ public static class Builder {
118152
private OzoneObj.StoreType storeType;
119153
private String volumeName;
120154
private String bucketName;
121-
private String keyName;
155+
private String name;
122156

123157
public static Builder newBuilder() {
124158
return new Builder();
@@ -145,14 +179,17 @@ public Builder setBucketName(String bucket) {
145179
}
146180

147181
public Builder setKeyName(String key) {
148-
this.keyName = key;
182+
this.name = key;
183+
return this;
184+
}
185+
186+
public Builder setPrefixName(String prefix) {
187+
this.name = prefix;
149188
return this;
150189
}
151190

152191
public OzoneObjInfo build() {
153-
return new OzoneObjInfo(resType, storeType, volumeName, bucketName,
154-
keyName);
192+
return new OzoneObjInfo(resType, storeType, volumeName, bucketName, name);
155193
}
156194
}
157-
158195
}

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/RadixTree.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -202,9 +202,15 @@ public String getLongestPrefix(String path) {
202202
break;
203203
}
204204
}
205-
return level >= 1 ?
206-
Paths.get(root.getName()).resolve(p.subpath(0, level)).toString() :
207-
root.getName();
205+
206+
if (level >= 1) {
207+
Path longestMatch =
208+
Paths.get(root.getName()).resolve(p.subpath(0, level));
209+
String ret = longestMatch.toString();
210+
return path.endsWith("/") ? ret + "/" : ret;
211+
} else {
212+
return root.getName();
213+
}
208214
}
209215

210216
// root of a radix tree has a name of "/" and may optionally has it value.

‎hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,7 @@ enum Status {
276276
NOT_A_FILE = 47;
277277
PERMISSION_DENIED = 48;
278278
TIMEOUT = 49;
279+
PREFIX_NOT_FOUND=50;
279280
}
280281

281282

@@ -507,15 +508,15 @@ message OzoneAclInfo {
507508
}
508509

509510
enum OzoneAclRights {
510-
READ = 1;
511-
WRITE = 2;
512-
CREATE = 3;
513-
LIST = 4;
514-
DELETE = 5;
515-
READ_ACL = 6;
516-
WRITE_ACL = 7;
517-
ALL = 8;
518-
NONE = 9;
511+
READ = 1;
512+
WRITE = 2;
513+
CREATE = 3;
514+
LIST = 4;
515+
DELETE = 5;
516+
READ_ACL = 6;
517+
WRITE_ACL = 7;
518+
ALL = 8;
519+
NONE = 9;
519520
}
520521
required OzoneAclType type = 1;
521522
required string name = 2;

‎hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/util/TestRadixTree.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,6 @@ public void testGetLongestPrefixPath() {
8484
assertEquals("g", lpn.getName());
8585
lpn.setValue(100);
8686

87-
8887
List<RadixNode<Integer>> lpq =
8988
ROOT.getLongestPrefixPath("/a/b/c/d/g/q");
9089
RadixNode<Integer> lqn = lpp.get(lpq.size()-1);
@@ -93,7 +92,6 @@ public void testGetLongestPrefixPath() {
9392
assertEquals("g", lqn.getName());
9493
assertEquals(100, (int)lqn.getValue());
9594

96-
9795
assertEquals("/a/", RadixTree.radixPathToString(
9896
ROOT.getLongestPrefixPath("/a/g")));
9997

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java‎

Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.io.IOException;
2222
import java.util.ArrayList;
2323
import java.util.Arrays;
24+
import java.util.BitSet;
2425
import java.util.HashMap;
2526
import java.util.Iterator;
2627
import java.util.List;
@@ -2200,6 +2201,66 @@ public void testNativeAclsForKey() throws Exception {
22002201
validateOzoneAcl(ozObj);
22012202
}
22022203

2204+
@Test
2205+
public void testNativeAclsForPrefix() throws Exception {
2206+
String volumeName = UUID.randomUUID().toString();
2207+
String bucketName = UUID.randomUUID().toString();
2208+
2209+
String prefix1 = "PF" + UUID.randomUUID().toString() + "/";
2210+
String key1 = prefix1 + "KEY" + UUID.randomUUID().toString();
2211+
2212+
String prefix2 = "PF" + UUID.randomUUID().toString() + "/";
2213+
String key2 = prefix2 + "KEY" + UUID.randomUUID().toString();
2214+
2215+
store.createVolume(volumeName);
2216+
OzoneVolume volume = store.getVolume(volumeName);
2217+
volume.createBucket(bucketName);
2218+
OzoneBucket bucket = volume.getBucket(bucketName);
2219+
assertNotNull("Bucket creation failed", bucket);
2220+
2221+
writeKey(key1, bucket);
2222+
writeKey(key2, bucket);
2223+
2224+
OzoneObj ozObj = new OzoneObjInfo.Builder()
2225+
.setVolumeName(volumeName)
2226+
.setBucketName(bucketName)
2227+
.setPrefixName(prefix1)
2228+
.setResType(OzoneObj.ResourceType.PREFIX)
2229+
.setStoreType(OzoneObj.StoreType.OZONE)
2230+
.build();
2231+
2232+
// add acl
2233+
BitSet aclRights1 = new BitSet();
2234+
aclRights1.set(ACLType.READ.ordinal());
2235+
OzoneAcl user1Acl = new OzoneAcl(ACLIdentityType.USER,
2236+
"user1", aclRights1);
2237+
assertTrue(store.addAcl(ozObj, user1Acl));
2238+
2239+
// get acl
2240+
List<OzoneAcl> aclsGet = store.getAcl(ozObj);
2241+
Assert.assertEquals(1, aclsGet.size());
2242+
Assert.assertEquals(user1Acl, aclsGet.get(0));
2243+
2244+
// remove acl
2245+
Assert.assertTrue(store.removeAcl(ozObj, user1Acl));
2246+
aclsGet = store.getAcl(ozObj);
2247+
Assert.assertEquals(0, aclsGet.size());
2248+
2249+
// set acl
2250+
BitSet aclRights2 = new BitSet();
2251+
aclRights2.set(ACLType.ALL.ordinal());
2252+
OzoneAcl group1Acl = new OzoneAcl(ACLIdentityType.GROUP,
2253+
"group1", aclRights2);
2254+
List<OzoneAcl> acls = new ArrayList<>();
2255+
acls.add(user1Acl);
2256+
acls.add(group1Acl);
2257+
Assert.assertTrue(store.setAcl(ozObj, acls));
2258+
2259+
// get acl
2260+
aclsGet = store.getAcl(ozObj);
2261+
Assert.assertEquals(2, aclsGet.size());
2262+
}
2263+
22032264
/**
22042265
* Helper function to get default acl list for current user.
22052266
*
@@ -2218,8 +2279,7 @@ private List<OzoneAcl> getAclList(OzoneConfiguration conf)
22182279
listOfAcls.add(new OzoneAcl(ACLIdentityType.USER,
22192280
ugi.getUserName(), userRights));
22202281
//Group ACLs of the User
2221-
List<String> userGroups = Arrays.asList(UserGroupInformation
2222-
.createRemoteUser(ugi.getUserName()).getGroupNames());
2282+
List<String> userGroups = Arrays.asList(ugi.getGroupNames());
22232283
userGroups.stream().forEach((group) -> listOfAcls.add(
22242284
new OzoneAcl(ACLIdentityType.GROUP, group, groupRights)));
22252285
return listOfAcls;

0 commit comments

Comments
 (0)