Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

kafka: support for multi-topic #4188

Closed
wants to merge 1 commit into from
Closed

kafka: support for multi-topic #4188

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 25 additions & 6 deletions cdc/sink/producer/kafka/kafka.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ type kafkaSaramaProducer struct {
closeCh chan struct{}
// atomic flag indicating whether the producer is closing
closing kafkaProducerClosingFlag
admin kafka.ClusterAdminClient
}

type kafkaProducerClosingFlag = int32
Expand All @@ -82,8 +83,25 @@ func (k *kafkaSaramaProducer) AsyncSendMessage(ctx context.Context, message *cod
return nil
}

topic := fmt.Sprintf("%s_%s_%s", k.topic, *message.Schema, *message.Table)

topics, err := k.admin.ListTopics()
if err != nil {
return cerror.WrapError(cerror.ErrKafkaNewSaramaProducer, err)
}

if _, exists := topics[topic]; !exists {
err = k.admin.CreateTopic(topic, &sarama.TopicDetail{
NumPartitions: 3,
ReplicationFactor: 1,
}, false)
if err != nil {
log.Error("Failed to create topic", zap.Error(err), zap.String("topic", topic))
}
}

msg := &sarama.ProducerMessage{
Topic: k.topic,
Topic: topic,
Key: sarama.ByteEncoder(message.Key),
Value: sarama.ByteEncoder(message.Value),
Partition: partition,
Expand Down Expand Up @@ -204,6 +222,11 @@ func (k *kafkaSaramaProducer) Close() error {
k.clientLock.Lock()
defer k.clientLock.Unlock()


if err := k.admin.Close(); err != nil {
log.Warn("close kafka cluster admin failed", zap.Error(err))
}

if k.producersReleased {
// We need to guard against double closing the clients,
// which could lead to panic.
Expand Down Expand Up @@ -274,11 +297,6 @@ func NewKafkaSaramaProducer(ctx context.Context, topic string, config *Config, o
if err != nil {
return nil, cerror.WrapError(cerror.ErrKafkaNewSaramaProducer, err)
}
defer func() {
if err := admin.Close(); err != nil {
log.Warn("close kafka cluster admin failed", zap.Error(err))
}
}()

if err := validateMaxMessageBytesAndCreateTopic(admin, topic, config, cfg, opts); err != nil {
return nil, cerror.WrapError(cerror.ErrKafkaNewSaramaProducer, err)
Expand Down Expand Up @@ -312,6 +330,7 @@ func NewKafkaSaramaProducer(ctx context.Context, topic string, config *Config, o
closeCh: make(chan struct{}),
failpointCh: make(chan error, 1),
closing: kafkaProducerRunning,
admin: admin,
}
go func() {
if err := k.run(ctx); err != nil && errors.Cause(err) != context.Canceled {
Expand Down