Skip to content

Commit 10036d5

Browse files
authored
PIP 110: Support Topic metadata - PART-1 create topic with properties (#12818)
Fixes #12629 ## Motivation The original discussion mail : https://lists.apache.org/thread/m9dkhq1fs6stsdwh78h84fsl5hs5v67f Introduce the ability to store metadata about topics. This would be very useful as with metadata you could add labels and other pieces of information that would allow defining the purpose of a topic, custom application-level properties. This feature will allow application-level diagnostic tools and maintenance tools to not need external databases to store such metadata. Imagine that we could add a simple key value map (String keys and String values) to the topic. These metadata could be set during topic creation and also updated.
1 parent 262c653 commit 10036d5

File tree

18 files changed

+437
-67
lines changed

18 files changed

+437
-67
lines changed

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java

+11
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ public class ManagedLedgerConfig {
7373
private int newEntriesCheckDelayInMillis = 10;
7474
private Clock clock = Clock.systemUTC();
7575
private ManagedLedgerInterceptor managedLedgerInterceptor;
76+
private Map<String, String> properties;
7677

7778
public boolean isCreateIfMissing() {
7879
return createIfMissing;
@@ -619,6 +620,16 @@ public void setBookKeeperEnsemblePlacementPolicyProperties(
619620
this.bookKeeperEnsemblePlacementPolicyProperties = bookKeeperEnsemblePlacementPolicyProperties;
620621
}
621622

623+
624+
public Map<String, String> getProperties() {
625+
return properties;
626+
}
627+
628+
629+
public void setProperties(Map<String, String> properties) {
630+
this.properties = properties;
631+
}
632+
622633
public boolean isDeletionAtBatchIndexLevelEnabled() {
623634
return deletionAtBatchIndexLevelEnabled;
624635
}

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -329,7 +329,8 @@ synchronized void initialize(final ManagedLedgerInitializeLedgerCallback callbac
329329
log.info("Opening managed ledger {}", name);
330330

331331
// Fetch the list of existing ledgers in the managed ledger
332-
store.getManagedLedgerInfo(name, config.isCreateIfMissing(), new MetaStoreCallback<ManagedLedgerInfo>() {
332+
store.getManagedLedgerInfo(name, config.isCreateIfMissing(), config.getProperties(),
333+
new MetaStoreCallback<ManagedLedgerInfo>() {
333334
@Override
334335
public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) {
335336
ledgersStat = stat;

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStore.java

+18-1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.apache.bookkeeper.mledger.impl;
2020

2121
import java.util.List;
22+
import java.util.Map;
2223
import java.util.concurrent.CompletableFuture;
2324
import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException;
2425
import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo;
@@ -51,7 +52,23 @@ interface MetaStoreCallback<T> {
5152
* whether the managed ledger metadata should be created if it doesn't exist already
5253
* @throws MetaStoreException
5354
*/
54-
void getManagedLedgerInfo(String ledgerName, boolean createIfMissing,
55+
default void getManagedLedgerInfo(String ledgerName, boolean createIfMissing,
56+
MetaStoreCallback<ManagedLedgerInfo> callback) {
57+
getManagedLedgerInfo(ledgerName, createIfMissing, null, callback);
58+
}
59+
60+
/**
61+
* Get the metadata used by the ManagedLedger.
62+
*
63+
* @param ledgerName
64+
* the name of the ManagedLedger
65+
* @param createIfMissing
66+
* whether the managed ledger metadata should be created if it doesn't exist already
67+
* @param properties
68+
* ledger properties
69+
* @throws MetaStoreException
70+
*/
71+
void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map<String, String> properties,
5572
MetaStoreCallback<ManagedLedgerInfo> callback);
5673

5774
/**

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java

+13-3
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import io.netty.buffer.Unpooled;
2525
import java.util.ArrayList;
2626
import java.util.List;
27+
import java.util.Map;
2728
import java.util.Optional;
2829
import java.util.concurrent.CompletableFuture;
2930
import java.util.concurrent.CompletionException;
@@ -81,7 +82,7 @@ public MetaStoreImpl(MetadataStore store, OrderedExecutor executor, String compr
8182
}
8283

8384
@Override
84-
public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing,
85+
public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map<String, String> properties,
8586
MetaStoreCallback<ManagedLedgerInfo> callback) {
8687
// Try to get the content or create an empty node
8788
String path = PREFIX + ledgerName;
@@ -103,8 +104,17 @@ public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing,
103104

104105
store.put(path, new byte[0], Optional.of(-1L))
105106
.thenAccept(stat -> {
106-
ManagedLedgerInfo info = ManagedLedgerInfo.getDefaultInstance();
107-
callback.operationComplete(info, stat);
107+
ManagedLedgerInfo.Builder ledgerBuilder = ManagedLedgerInfo.newBuilder();
108+
if (properties != null) {
109+
properties.forEach((k, v) -> {
110+
ledgerBuilder.addProperties(
111+
MLDataFormats.KeyValue.newBuilder()
112+
.setKey(k)
113+
.setValue(v)
114+
.build());
115+
});
116+
}
117+
callback.operationComplete(ledgerBuilder.build(), stat);
108118
}).exceptionally(ex -> {
109119
callback.operationFailed(getException(ex));
110120
return null;

managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java

+30
Original file line numberDiff line numberDiff line change
@@ -1632,6 +1632,36 @@ public void cursorReadsWithDiscardedEmptyLedgers() throws Exception {
16321632
assertEquals(c1.readEntries(1).size(), 0);
16331633
}
16341634

1635+
@Test
1636+
public void testSetTopicMetadata() throws Exception {
1637+
Map<String, String> properties = new HashMap<>();
1638+
properties.put("key1", "value1");
1639+
properties.put("key2", "value2");
1640+
final MetaStore store = factory.getMetaStore();
1641+
final CountDownLatch latch = new CountDownLatch(1);
1642+
final ManagedLedgerInfo[] storedMLInfo = new ManagedLedgerInfo[1];
1643+
store.getManagedLedgerInfo("my_test_ledger", true, properties, new MetaStoreCallback<ManagedLedgerInfo>() {
1644+
@Override
1645+
public void operationComplete(ManagedLedgerInfo result, Stat version) {
1646+
storedMLInfo[0] = result;
1647+
latch.countDown();
1648+
}
1649+
1650+
@Override
1651+
public void operationFailed(MetaStoreException e) {
1652+
latch.countDown();
1653+
fail("Should have failed here");
1654+
}
1655+
});
1656+
latch.await();
1657+
1658+
assertEquals(storedMLInfo[0].getPropertiesCount(), 2);
1659+
assertEquals(storedMLInfo[0].getPropertiesList().get(0).getKey(), "key1");
1660+
assertEquals(storedMLInfo[0].getPropertiesList().get(0).getValue(), "value1");
1661+
assertEquals(storedMLInfo[0].getPropertiesList().get(1).getKey(), "key2");
1662+
assertEquals(storedMLInfo[0].getPropertiesList().get(1).getValue(), "value2");
1663+
}
1664+
16351665
@Test
16361666
public void cursorReadsWithDiscardedEmptyLedgersStillListed() throws Exception {
16371667
ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger");

pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java

+12-6
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import com.google.errorprone.annotations.CanIgnoreReturnValue;
2323
import java.util.ArrayList;
2424
import java.util.List;
25+
import java.util.Map;
2526
import java.util.Optional;
2627
import java.util.Set;
2728
import java.util.concurrent.CompletableFuture;
@@ -569,6 +570,11 @@ protected List<String> getTopicPartitionList(TopicDomain topicDomain) {
569570

570571
protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int numPartitions,
571572
boolean createLocalTopicOnly) {
573+
internalCreatePartitionedTopic(asyncResponse, numPartitions, createLocalTopicOnly, null);
574+
}
575+
576+
protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int numPartitions,
577+
boolean createLocalTopicOnly, Map<String, String> properties) {
572578
Integer maxTopicsPerNamespace = null;
573579

574580
try {
@@ -635,7 +641,7 @@ protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int n
635641
return;
636642
}
637643

638-
provisionPartitionedTopicPath(asyncResponse, numPartitions, createLocalTopicOnly)
644+
provisionPartitionedTopicPath(asyncResponse, numPartitions, createLocalTopicOnly, properties)
639645
.thenCompose(ignored -> tryCreatePartitionsAsync(numPartitions))
640646
.whenComplete((ignored, ex) -> {
641647
if (ex != null) {
@@ -674,7 +680,7 @@ protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int n
674680
((TopicsImpl) pulsar().getBrokerService()
675681
.getClusterPulsarAdmin(cluster, clusterDataOp).topics())
676682
.createPartitionedTopicAsync(
677-
topicName.getPartitionedTopicName(), numPartitions, true);
683+
topicName.getPartitionedTopicName(), numPartitions, true, null);
678684
})
679685
.exceptionally(throwable -> {
680686
log.error("Failed to create partition topic in cluster {}.", cluster, throwable);
@@ -713,13 +719,13 @@ protected CompletableFuture<Boolean> checkTopicExistsAsync(TopicName topicName)
713719
});
714720
}
715721

716-
private CompletableFuture<Void> provisionPartitionedTopicPath(AsyncResponse asyncResponse,
717-
int numPartitions,
718-
boolean createLocalTopicOnly) {
722+
private CompletableFuture<Void> provisionPartitionedTopicPath(AsyncResponse asyncResponse, int numPartitions,
723+
boolean createLocalTopicOnly,
724+
Map<String, String> properties) {
719725
CompletableFuture<Void> future = new CompletableFuture<>();
720726
namespaceResources()
721727
.getPartitionedTopicResources()
722-
.createPartitionedTopicAsync(topicName, new PartitionedTopicMetadata(numPartitions))
728+
.createPartitionedTopicAsync(topicName, new PartitionedTopicMetadata(numPartitions, properties))
723729
.whenComplete((ignored, ex) -> {
724730
if (ex != null) {
725731
if (ex instanceof AlreadyExistsException) {

pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java

+8-4
Original file line numberDiff line numberDiff line change
@@ -370,7 +370,7 @@ protected void internalRevokePermissionsOnTopic(String role) {
370370
revokePermissions(topicName.toString(), role);
371371
}
372372

373-
protected void internalCreateNonPartitionedTopic(boolean authoritative) {
373+
protected void internalCreateNonPartitionedTopic(boolean authoritative, Map<String, String> properties) {
374374
validateNonPartitionTopicName(topicName.getLocalName());
375375
if (topicName.isGlobal()) {
376376
validateGlobalNamespaceOwnership(namespaceName);
@@ -391,7 +391,7 @@ protected void internalCreateNonPartitionedTopic(boolean authoritative) {
391391
throw new RestException(Status.CONFLICT, "This topic already exists");
392392
}
393393

394-
Topic createdTopic = getOrCreateTopic(topicName);
394+
Topic createdTopic = getOrCreateTopic(topicName, properties);
395395
log.info("[{}] Successfully created non-partitioned topic {}", clientAppId(), createdTopic);
396396
} catch (Exception e) {
397397
if (e instanceof RestException) {
@@ -3891,8 +3891,12 @@ private CompletableFuture<Topic> topicNotFoundReasonAsync(TopicName topicName) {
38913891
}
38923892

38933893
private Topic getOrCreateTopic(TopicName topicName) {
3894-
return pulsar().getBrokerService().getTopic(
3895-
topicName.toString(), true).thenApply(Optional::get).join();
3894+
return getOrCreateTopic(topicName, null);
3895+
}
3896+
3897+
private Topic getOrCreateTopic(TopicName topicName, Map<String, String> properties) {
3898+
return pulsar().getBrokerService().getTopic(topicName.toString(), true, properties)
3899+
.thenApply(Optional::get).join();
38963900
}
38973901

38983902
/**

pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ public void createNonPartitionedTopic(
201201
validateNamespaceName(tenant, cluster, namespace);
202202
validateTopicName(tenant, cluster, namespace, encodedTopic);
203203
validateGlobalNamespaceOwnership();
204-
internalCreateNonPartitionedTopic(authoritative);
204+
internalCreateNonPartitionedTopic(authoritative, null);
205205
}
206206

207207
/**

pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -264,12 +264,14 @@ public void createNonPartitionedTopic(
264264
@ApiParam(value = "Specify topic name", required = true)
265265
@PathParam("topic") @Encoded String encodedTopic,
266266
@ApiParam(value = "Is authentication required to perform this operation")
267-
@QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
267+
@QueryParam("authoritative") @DefaultValue("false") boolean authoritative,
268+
@ApiParam(value = "Key value pair properties for the topic metadata")
269+
Map<String, String> properties) {
268270
validateNamespaceName(tenant, namespace);
269271
validateGlobalNamespaceOwnership();
270272
validateTopicName(tenant, namespace, encodedTopic);
271273
validateCreateTopic(topicName);
272-
internalCreateNonPartitionedTopic(authoritative);
274+
internalCreateNonPartitionedTopic(authoritative, properties);
273275
}
274276

275277
@GET
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.pulsar.broker.admin.v3;
20+
21+
import io.swagger.annotations.Api;
22+
import io.swagger.annotations.ApiOperation;
23+
import io.swagger.annotations.ApiParam;
24+
import io.swagger.annotations.ApiResponse;
25+
import io.swagger.annotations.ApiResponses;
26+
import javax.ws.rs.DefaultValue;
27+
import javax.ws.rs.Encoded;
28+
import javax.ws.rs.PUT;
29+
import javax.ws.rs.Path;
30+
import javax.ws.rs.PathParam;
31+
import javax.ws.rs.Produces;
32+
import javax.ws.rs.QueryParam;
33+
import javax.ws.rs.container.AsyncResponse;
34+
import javax.ws.rs.container.Suspended;
35+
import javax.ws.rs.core.MediaType;
36+
import org.apache.pulsar.broker.admin.impl.PersistentTopicsBase;
37+
import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
38+
import org.apache.pulsar.common.policies.data.PolicyName;
39+
import org.apache.pulsar.common.policies.data.PolicyOperation;
40+
import org.slf4j.Logger;
41+
import org.slf4j.LoggerFactory;
42+
43+
/**
44+
*/
45+
@Path("/persistent")
46+
@Produces(MediaType.APPLICATION_JSON)
47+
@Api(value = "/persistent", description = "Persistent topic admin apis", tags = "persistent topic")
48+
public class PersistentTopics extends PersistentTopicsBase {
49+
50+
@PUT
51+
@Path("/{tenant}/{namespace}/{topic}/partitions")
52+
@ApiOperation(value = "Create a partitioned topic.",
53+
notes = "It needs to be called before creating a producer on a partitioned topic.")
54+
@ApiResponses(value = {
55+
@ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"),
56+
@ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"),
57+
@ApiResponse(code = 403, message = "Don't have admin permission"),
58+
@ApiResponse(code = 404, message = "Tenant does not exist"),
59+
@ApiResponse(code = 406, message = "The number of partitions should be more than 0 and"
60+
+ " less than or equal to maxNumPartitionsPerPartitionedTopic"),
61+
@ApiResponse(code = 409, message = "Partitioned topic already exist"),
62+
@ApiResponse(code = 412,
63+
message = "Failed Reason : Name is invalid or Namespace does not have any clusters configured"),
64+
@ApiResponse(code = 500, message = "Internal server error"),
65+
@ApiResponse(code = 503, message = "Failed to validate global cluster configuration")
66+
})
67+
public void createPartitionedTopic(
68+
@Suspended final AsyncResponse asyncResponse,
69+
@ApiParam(value = "Specify the tenant", required = true)
70+
@PathParam("tenant") String tenant,
71+
@ApiParam(value = "Specify the namespace", required = true)
72+
@PathParam("namespace") String namespace,
73+
@ApiParam(value = "Specify topic name", required = true)
74+
@PathParam("topic") @Encoded String encodedTopic,
75+
@ApiParam(value = "The metadata for the topic",
76+
required = true, type = "PartitionedTopicMetadata") PartitionedTopicMetadata metadata,
77+
@QueryParam("createLocalTopicOnly") @DefaultValue("false") boolean createLocalTopicOnly) {
78+
try {
79+
validateNamespaceName(tenant, namespace);
80+
validateGlobalNamespaceOwnership();
81+
validatePartitionedTopicName(tenant, namespace, encodedTopic);
82+
validateTopicPolicyOperation(topicName, PolicyName.PARTITION, PolicyOperation.WRITE);
83+
validateCreateTopic(topicName);
84+
internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly,
85+
metadata.properties);
86+
} catch (Exception e) {
87+
log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e);
88+
resumeAsyncResponseExceptionally(asyncResponse, e);
89+
}
90+
}
91+
92+
private static final Logger log = LoggerFactory.getLogger(PersistentTopics.class);
93+
}

0 commit comments

Comments
 (0)