-
Notifications
You must be signed in to change notification settings - Fork 4.4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
ca: fix a bug that caused a non blocking leaf cert query after a blocking leaf cert query to block #12820
ca: fix a bug that caused a non blocking leaf cert query after a blocking leaf cert query to block #12820
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
```release-note:bug | ||
ca: fix a bug that caused a non blocking leaf cert query after a blocking leaf cert query to block | ||
``` |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -6202,6 +6202,101 @@ func TestAgentConnectCALeafCert_goodNotLocal(t *testing.T) { | |
} | ||
} | ||
|
||
func TestAgentConnectCALeafCert_nonBlockingQuery_after_blockingQuery_shouldNotBlock(t *testing.T) { | ||
// see: https://github.com/hashicorp/consul/issues/12048 | ||
|
||
runStep := func(t *testing.T, name string, fn func(t *testing.T)) { | ||
t.Helper() | ||
if !t.Run(name, fn) { | ||
t.FailNow() | ||
} | ||
} | ||
|
||
if testing.Short() { | ||
t.Skip("too slow for testing.Short") | ||
} | ||
|
||
t.Parallel() | ||
|
||
a := NewTestAgent(t, "") | ||
testrpc.WaitForTestAgent(t, a.RPC, "dc1") | ||
testrpc.WaitForActiveCARoot(t, a.RPC, "dc1", nil) | ||
|
||
{ | ||
// Register a local service | ||
args := &structs.ServiceDefinition{ | ||
ID: "foo", | ||
Name: "test", | ||
Address: "127.0.0.1", | ||
Port: 8000, | ||
Check: structs.CheckType{ | ||
TTL: 15 * time.Second, | ||
}, | ||
} | ||
req := httptest.NewRequest("PUT", "/v1/agent/service/register", jsonReader(args)) | ||
resp := httptest.NewRecorder() | ||
a.srv.h.ServeHTTP(resp, req) | ||
if !assert.Equal(t, 200, resp.Code) { | ||
t.Log("Body: ", resp.Body.String()) | ||
} | ||
} | ||
|
||
var ( | ||
serialNumber string | ||
index string | ||
issued structs.IssuedCert | ||
) | ||
runStep(t, "do initial non-blocking query", func(t *testing.T) { | ||
req := httptest.NewRequest("GET", "/v1/agent/connect/ca/leaf/test", nil) | ||
resp := httptest.NewRecorder() | ||
a.srv.h.ServeHTTP(resp, req) | ||
|
||
dec := json.NewDecoder(resp.Body) | ||
require.NoError(t, dec.Decode(&issued)) | ||
serialNumber = issued.SerialNumber | ||
|
||
require.Equal(t, "MISS", resp.Header().Get("X-Cache"), | ||
"for the leaf cert cache type these are always MISS") | ||
index = resp.Header().Get("X-Consul-Index") | ||
}) | ||
|
||
ctx, cancel := context.WithCancel(context.Background()) | ||
defer cancel() | ||
go func() { | ||
// launch goroutine for blocking query | ||
req := httptest.NewRequest("GET", "/v1/agent/connect/ca/leaf/test?index="+index, nil).Clone(ctx) | ||
resp := httptest.NewRecorder() | ||
a.srv.h.ServeHTTP(resp, req) | ||
}() | ||
|
||
// We just need to ensure that the above blocking query is in-flight before | ||
// the next step, so do a little sleep. | ||
time.Sleep(50 * time.Millisecond) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could we make this deterministic with context.WithTimeout and calling the above code serially? I'm fine with merging without; just calling out potential alternatives to sleeping There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Much like all of the other blocking query tests, the trick is figuring out how to initiate a blocking query that does not return, know it has reached the important part, and only THEN issue your followup query. We don't have any signaling mechanism to know exactly when something is actually blocking. |
||
|
||
// The initial non-blocking query populated the leaf cert cache entry | ||
// implicitly. The agent cache doesn't prune entries very often at all, so | ||
// in between both of these steps the data should still be there, causing | ||
// this to be a HIT that completes in less than 10m (the default inner leaf | ||
// cert blocking query timeout). | ||
runStep(t, "do a non-blocking query that should not block", func(t *testing.T) { | ||
rboyer marked this conversation as resolved.
Show resolved
Hide resolved
|
||
req := httptest.NewRequest("GET", "/v1/agent/connect/ca/leaf/test", nil) | ||
resp := httptest.NewRecorder() | ||
a.srv.h.ServeHTTP(resp, req) | ||
|
||
var issued2 structs.IssuedCert | ||
dec := json.NewDecoder(resp.Body) | ||
require.NoError(t, dec.Decode(&issued2)) | ||
|
||
require.Equal(t, "HIT", resp.Header().Get("X-Cache")) | ||
|
||
// If this is actually returning a cached result, the serial number | ||
// should be unchanged. | ||
require.Equal(t, serialNumber, issued2.SerialNumber) | ||
|
||
require.Equal(t, issued, issued2) | ||
}) | ||
} | ||
|
||
func TestAgentConnectCALeafCert_Vault_doesNotChurnLeafCertsAtIdle(t *testing.T) { | ||
ca.SkipIfVaultNotPresent(t) | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -376,6 +376,13 @@ func (c *Cache) getEntryLocked( | |
// Check if re-validate is requested. If so the first time round the | ||
// loop is not a hit but subsequent ones should be treated normally. | ||
if !tEntry.Opts.Refresh && info.MustRevalidate { | ||
if entry.Fetching { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ah, this is one of those lines that I hope doesn't surface code that was relying on side effects here. AFAIK, (and I don't know much) there's not a lot of cache types that actually use So my gut tells me that this should work beautifully for the bug in #12048 🤞🏼 But it would feel good to check in with @banks or @freddygv if they have more context here to get a third/ fourth pair of eyes 💯 |
||
// There is an active blocking query for this data, which has not | ||
// returned. We can logically deduce that the contents of the cache | ||
// are actually current, and we can simply return this while | ||
// leaving the blocking query alone. | ||
return true, true, entry | ||
} | ||
return true, false, entry | ||
} | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this is a non actionable comment:
for any of y'all wondering about testing that the blocking query does what we want for the leaf cert, that use case is here: https://github.com/hashicorp/consul/blob/main/agent/agent_endpoint_test.go#L6016-L6033