-
Notifications
You must be signed in to change notification settings - Fork 1.3k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(spanner): add support of using multiplexed session with ReadOnlyTransactions #10269
Changes from 16 commits
21c837f
8de2d37
2849e1c
e41d900
8d1ac64
37023f5
e9eb4ba
2ff8613
fc55bc9
1602470
c835989
0b63e8d
c58eff5
ddfab86
6356f0e
a1160db
3b45b8e
7941056
0a746f9
6cb0b11
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,111 @@ | ||
#!/bin/bash | ||
# Copyright 2024 Google LLC | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License.. | ||
|
||
# Fail on any error | ||
set -eo pipefail | ||
|
||
# Display commands being run | ||
set -x | ||
|
||
# cd to project dir on Kokoro instance | ||
cd github/google-cloud-go | ||
|
||
go version | ||
|
||
export GOCLOUD_HOME=$KOKORO_ARTIFACTS_DIR/google-cloud-go/ | ||
export PATH="$GOPATH/bin:$PATH" | ||
export GO111MODULE=on | ||
export GOPROXY=https://proxy.golang.org | ||
export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/${GOOGLE_APPLICATION_CREDENTIALS} | ||
# Move code into artifacts dir | ||
mkdir -p $GOCLOUD_HOME | ||
git clone . $GOCLOUD_HOME | ||
cd $GOCLOUD_HOME | ||
|
||
try3() { eval "$*" || eval "$*" || eval "$*"; } | ||
|
||
# All packages, including +build tools, are fetched. | ||
try3 go mod download | ||
|
||
set +e # Run all tests, don't stop after the first failure. | ||
exit_code=0 | ||
|
||
case $JOB_TYPE in | ||
integration-with-multiplexed-session ) | ||
GOOGLE_CLOUD_SPANNER_MULTIPLEXED_SESSIONS=true | ||
echo "running presubmit with multiplexed sessions enabled: $GOOGLE_CLOUD_SPANNER_MULTIPLEXED_SESSIONS" | ||
;; | ||
esac | ||
|
||
# Run tests in the current directory and tee output to log file, | ||
# to be pushed to GCS as artifact. | ||
runPresubmitTests() { | ||
if [[ $PWD == *"/internal/"* ]] || | ||
[[ $PWD == *"/third_party/"* ]]; then | ||
# internal tools only expected to work with latest go version | ||
return | ||
fi | ||
|
||
if [ -z ${RUN_INTEGRATION_TESTS} ]; then | ||
GOWORK=off go test -race -v -timeout 15m -short ./... 2>&1 | | ||
tee sponge_log.log | ||
else | ||
GOWORK=off go test -race -v -timeout 45m ./... 2>&1 | | ||
tee sponge_log.log | ||
fi | ||
|
||
# Skip running integration tests since Emulator does not support Multiplexed sessions | ||
# Run integration tests against an emulator. | ||
# if [ -f "emulator_test.sh" ]; then | ||
# ./emulator_test.sh | ||
# fi | ||
# Takes the kokoro output log (raw stdout) and creates a machine-parseable | ||
# xUnit XML file. | ||
cat sponge_log.log | | ||
go-junit-report -set-exit-code >sponge_log.xml | ||
# Add the exit codes together so we exit non-zero if any module fails. | ||
exit_code=$(($exit_code + $?)) | ||
if [[ $PWD != *"/internal/"* ]]; then | ||
GOWORK=off go build ./... | ||
fi | ||
exit_code=$(($exit_code + $?)) | ||
} | ||
|
||
SIGNIFICANT_CHANGES=$(git --no-pager diff --name-only origin/main...$KOKORO_GIT_COMMIT_google_cloud_go | | ||
grep -Ev '(\.md$|^\.github|\.json$|\.yaml$)' | xargs dirname | sort -u || true) | ||
|
||
if [ -z $SIGNIFICANT_CHANGES ]; then | ||
echo "No changes detected, skipping tests" | ||
exit 0 | ||
fi | ||
|
||
# CHANGED_DIRS is the list of significant top-level directories that changed, | ||
# but weren't deleted by the current PR. CHANGED_DIRS will be empty when run on main. | ||
CHANGED_DIRS=$(echo "$SIGNIFICANT_CHANGES" | tr ' ' '\n' | cut -d/ -f1 | sort -u | | ||
tr '\n' ' ' | xargs ls -d 2>/dev/null || true) | ||
|
||
echo "Running tests only in changed submodules: $CHANGED_DIRS" | ||
for d in $CHANGED_DIRS; do | ||
# run tests only if spanner module is part of $CHANGED_DIRS | ||
if [[ $CHANGED_DIRS =~ spanner ]];then | ||
for i in $(find "$d" -name go.mod); do | ||
pushd $(dirname $i) | ||
runPresubmitTests | ||
popd | ||
done | ||
fi | ||
done | ||
|
||
exit $exit_code |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -52,6 +52,19 @@ func TestOCStats(t *testing.T) { | |
func TestOCStats_SessionPool(t *testing.T) { | ||
skipForPGTest(t) | ||
DisableGfeLatencyAndHeaderMissingCountViews() | ||
// expectedValues is a map of expected values for different configurations of | ||
// multiplexed session env="GOOGLE_CLOUD_SPANNER_MULTIPLEXED_SESSIONS". | ||
expectedValues := map[string]map[bool]string{ | ||
"open_session_count": { | ||
false: "25", | ||
// since we are doing only R/O operations and MinOpened=0, we should have only one session. | ||
true: "1", | ||
}, | ||
"max_in_use_sessions": { | ||
false: "1", | ||
true: "0", | ||
}, | ||
} | ||
for _, test := range []struct { | ||
name string | ||
view *view.View | ||
|
@@ -62,7 +75,7 @@ func TestOCStats_SessionPool(t *testing.T) { | |
"OpenSessionCount", | ||
OpenSessionCountView, | ||
"open_session_count", | ||
"25", | ||
expectedValues["open_session_count"][isMultiplexEnabled], | ||
}, | ||
{ | ||
"MaxAllowedSessionsCount", | ||
|
@@ -74,7 +87,7 @@ func TestOCStats_SessionPool(t *testing.T) { | |
"MaxInUseSessionsCount", | ||
MaxInUseSessionsCountView, | ||
"max_in_use_sessions", | ||
"1", | ||
expectedValues["max_in_use_sessions"][isMultiplexEnabled], | ||
}, | ||
{ | ||
"AcquiredSessionsCount", | ||
|
@@ -167,11 +180,15 @@ func TestOCStats_SessionPool_SessionsCount(t *testing.T) { | |
}) | ||
client.Single().ReadRow(context.Background(), "Users", Key{"alice"}, []string{"email"}) | ||
|
||
expectedSpans := 2 | ||
if isMultiplexEnabled { | ||
expectedSpans = 1 | ||
} | ||
// Wait for a while to see all exported metrics. | ||
waitFor(t, func() error { | ||
select { | ||
case stat := <-te.Stats: | ||
if len(stat.Rows) >= 2 { | ||
if len(stat.Rows) >= expectedSpans { | ||
return nil | ||
} | ||
} | ||
|
@@ -183,7 +200,7 @@ func TestOCStats_SessionPool_SessionsCount(t *testing.T) { | |
case stat := <-te.Stats: | ||
// There are 4 types for this metric, so we should see at least four | ||
// rows. | ||
if len(stat.Rows) < 2 { | ||
if len(stat.Rows) < expectedSpans { | ||
t.Fatal("No enough metrics are exported") | ||
} | ||
if got, want := stat.View.Measure.Name(), statsPrefix+"num_sessions_in_pool"; got != want { | ||
|
@@ -216,6 +233,9 @@ func TestOCStats_SessionPool_SessionsCount(t *testing.T) { | |
} | ||
|
||
func TestOCStats_SessionPool_GetSessionTimeoutsCount(t *testing.T) { | ||
if isMultiplexEnabled { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think we should skip this test when multiplexed sessions are enabled (or otherwise, we should add a separate test for multiplexed sessions). This test verifies the behavior of the client if for whatever reason the creation of session(s) is taking longer than expected. That could also happen with multiplexed sessions. |
||
t.Skip("Skipping test as multiplexed sessions will be available from background thread if enabled as soon as client is created") | ||
} | ||
DisableGfeLatencyAndHeaderMissingCountViews() | ||
te := testutil.NewTestExporter(GetSessionTimeoutsCountView) | ||
defer te.Unregister() | ||
|
@@ -227,7 +247,10 @@ func TestOCStats_SessionPool_GetSessionTimeoutsCount(t *testing.T) { | |
stestutil.SimulatedExecutionTime{ | ||
MinimumExecutionTime: 2 * time.Millisecond, | ||
}) | ||
|
||
server.TestSpanner.PutExecutionTime(stestutil.MethodCreateSession, | ||
stestutil.SimulatedExecutionTime{ | ||
MinimumExecutionTime: 2 * time.Millisecond, | ||
}) | ||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) | ||
defer cancel() | ||
client.Single().ReadRow(ctx, "Users", Key{"alice"}, []string{"email"}) | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: Would you mind adding a comment here that explains which span will be skipped when multiplexed sessions are enabled?