-
Notifications
You must be signed in to change notification settings - Fork 1.8k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Do not use partition cache for unknown topics #372
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -65,6 +65,60 @@ func TestCachedPartitions(t *testing.T) { | |
safeClose(t, client) | ||
} | ||
|
||
func TestClientDoesntCachePartitionsForTopicsWithErrors(t *testing.T) { | ||
seedBroker := newMockBroker(t, 1) | ||
|
||
replicas := []int32{seedBroker.BrokerID()} | ||
|
||
metadataResponse := new(MetadataResponse) | ||
metadataResponse.AddBroker(seedBroker.Addr(), seedBroker.BrokerID()) | ||
metadataResponse.AddTopicPartition("my_topic", 1, replicas[0], replicas, replicas, ErrNoError) | ||
metadataResponse.AddTopicPartition("my_topic", 2, replicas[0], replicas, replicas, ErrNoError) | ||
seedBroker.Returns(metadataResponse) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this response can just be empty, there's no need for these There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ah I see you actually test this later, nevermind There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. broker is still not necessary in any of these responses There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I thouhgt it was a good idea to ensure we are now not always busting the cache. |
||
|
||
config := NewConfig() | ||
config.Metadata.Retry.Max = 0 | ||
client, err := NewClient([]string{seedBroker.Addr()}, config) | ||
if err != nil { | ||
t.Fatal(err) | ||
} | ||
|
||
metadataResponse = new(MetadataResponse) | ||
metadataResponse.AddTopic("unknown", ErrUnknownTopicOrPartition) | ||
seedBroker.Returns(metadataResponse) | ||
|
||
partitions, err := client.Partitions("unknown") | ||
|
||
if err != ErrUnknownTopicOrPartition { | ||
t.Error("Expected ErrUnknownTopicOrPartition, found", err) | ||
} | ||
if partitions != nil { | ||
t.Errorf("Should return nil as partition list, found %v", partitions) | ||
} | ||
|
||
// Should still use the cache of a known topic | ||
partitions, err = client.Partitions("my_topic") | ||
if err != nil { | ||
t.Errorf("Expected no error, found %v", err) | ||
} | ||
|
||
metadataResponse = new(MetadataResponse) | ||
metadataResponse.AddTopic("unknown", ErrUnknownTopicOrPartition) | ||
seedBroker.Returns(metadataResponse) | ||
|
||
// Should not use cache for unknown topic | ||
partitions, err = client.Partitions("unknown") | ||
if err != ErrUnknownTopicOrPartition { | ||
t.Error("Expected ErrUnknownTopicOrPartition, found", err) | ||
} | ||
if partitions != nil { | ||
t.Errorf("Should return nil as partition list, found %v", partitions) | ||
} | ||
|
||
seedBroker.Close() | ||
safeClose(t, client) | ||
} | ||
|
||
func TestClientSeedBrokers(t *testing.T) { | ||
seedBroker := newMockBroker(t, 1) | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -182,25 +182,31 @@ func (m *MetadataResponse) AddBroker(addr string, id int32) { | |
m.Brokers = append(m.Brokers, &Broker{id: id, addr: addr}) | ||
} | ||
|
||
func (m *MetadataResponse) AddTopicPartition(topic string, partition, brokerID int32, replicas, isr []int32, err KError) { | ||
var match *TopicMetadata | ||
func (m *MetadataResponse) AddTopic(topic string, err KError) *TopicMetadata { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Adding an API requires a minor version bump in semver (1.1.0 vs 1.0.1). I'm torn between just making this private or eating the version bump. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's just do the version bump; there's no good reason this method should be hidden if There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should we doc the new API in the changelog then? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't really feel the need to; it will just show up in the godoc. |
||
var tmatch *TopicMetadata | ||
|
||
for _, tm := range m.Topics { | ||
if tm.Name == topic { | ||
match = tm | ||
tmatch = tm | ||
goto foundTopic | ||
} | ||
} | ||
|
||
match = new(TopicMetadata) | ||
match.Name = topic | ||
m.Topics = append(m.Topics, match) | ||
tmatch = new(TopicMetadata) | ||
tmatch.Name = topic | ||
m.Topics = append(m.Topics, tmatch) | ||
|
||
foundTopic: | ||
|
||
tmatch.Err = err | ||
return tmatch | ||
} | ||
|
||
func (m *MetadataResponse) AddTopicPartition(topic string, partition, brokerID int32, replicas, isr []int32, err KError) { | ||
tmatch := m.AddTopic(topic, ErrNoError) | ||
var pmatch *PartitionMetadata | ||
|
||
for _, pm := range match.Partitions { | ||
for _, pm := range tmatch.Partitions { | ||
if pm.ID == partition { | ||
pmatch = pm | ||
goto foundPartition | ||
|
@@ -209,7 +215,7 @@ foundTopic: | |
|
||
pmatch = new(PartitionMetadata) | ||
pmatch.ID = partition | ||
match.Partitions = append(match.Partitions, pmatch) | ||
tmatch.Partitions = append(tmatch.Partitions, pmatch) | ||
|
||
foundPartition: | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This makes this method return
ErrInvalidTopic
, even if all the other topics are OK.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
which is fine, I think - we only get this if you explicitly pass in "SomeInvalidTopic\" to
RefreshMetadata
, so returningErrInvalidTopic
in that case is correct, even if not all topics were invalidThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
In that case we should probably also set the error on
ErrUnknownTopicOrPartition
, so that gets returned to the user as well.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ah, I thought we checked the error return before the retry return, but we do not, so this is OK, good catch 👍
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Fixed.