From 667d08121567b2a3b6133b818ff6f2900b9ebc98 Mon Sep 17 00:00:00 2001 From: Vasil Averyanau Date: Fri, 22 Nov 2024 09:35:27 +0100 Subject: [PATCH] fix: handle ssl only scylla cluster setup (#4114) * fix: adds SSL_ENABLED flag to start scylla cluster in ssl only mode This adds SSL_ENABLED flag to Makefile, so that when you run SSL_ENABLED=true make start-dev-env the scylla cluster will be created with ssl_only config. * fix: handle ssl only scylla clusters This fixes how SM decides which port to use when connecting to Scylla nodes. * fix: CQLAddr provides ssl or non-ssl addr depending on cluster conf. This replaces CQLAddr and CQLSSLAddr with one function which returns correct cql addr depending on cluster configuration. Also backup worker is modified a little bit to get cluster configuration with tls related info. * fix(Makefile): use yq to produce scylla config with ssl enabled This uses yq to delete non ssl port from scylla.yaml config and also merges it with scylla-ssl.yaml which contains requried parameters to enable ssl in scylla cluster. * fix: typo in testing/scylla/config/scylla-ssl.yaml Co-authored-by: karol-kokoszka <50329145+karol-kokoszka@users.noreply.github.com> * fix(test): use scylla cluster with SSL for integration tests This enables ssl only scylla cluster for the most of our integration tests in ci. This also fixes cqlping test so it supports a scylla cluster with ssl. * fix(cluster): simplifies SingleHostSessionOption when dealing with SSL This changes the signature of SessionConfigOption so that SingleHostSession func can be simplified when Scylla cluster uses SSL. * fix(test): adds ssl support to cqlping integration tests This adds ssl related configuration options to cqlping integration tests config when ssl is enabled. * fix(test): adds ssl support to repair integration test This adds ssl support to repair integartion test case that uses cqlping * fix(test): adds ssl support to healthcheck integration tests * fix(test): unifies how SSL_ENABLED is used in testconfig * fix(ci): adds missing ssl-enabled option for a one entry in ci config * refactor: moves parsing of SSL_ENABLED env var to the testconfig pkg This refactor some parts of the tests that are using SSL_ENABLED env var. * fix(test): use cqlping with ssl for the restore test of old scylla ver This fixes how restore integration tests handle old Scylla versions: old versions require a restart after schema restoration. To ensure Scylla is up and running, the tests perform a CQL ping, which should be initialized correctly when SSL is enabled. --------- Co-authored-by: karol-kokoszka <50329145+karol-kokoszka@users.noreply.github.com> (cherry picked from commit 75fb75c89c8551a0d6b8cdca9ae091dd26331d62) --- .github/actions/test-setup/action.yml | 8 +- .github/cfg/integration-test-cfg.yaml | 12 +- .github/cfg/integration-test-core.yaml | 19 ++- .github/cfg/main.go | 5 + ...ation-tests-2023.1.11-IPV4-raftschema.yaml | 17 ++- .../integration-tests-2023.1.11-IPV4.yaml | 17 ++- ...ation-tests-2023.1.11-IPV6-raftschema.yaml | 17 ++- .../integration-tests-2024.1.12-IPV4.yaml | 17 ++- .../integration-tests-2024.1.12-IPV6.yaml | 17 ++- .../integration-tests-6.2.0-IPV4-tablets.yaml | 17 ++- .../integration-tests-6.2.0-IPV4.yaml | 17 ++- ...ation-tests-6.2.0-IPV6-tablets-nossl.yaml} | 19 ++- ...latest-enterprise-IPV4-tablets-nossl.yaml} | 19 ++- ...egration-tests-latest-enterprise-IPV4.yaml | 17 ++- Makefile | 5 +- README.md | 13 +- pkg/ping/cqlping/cqlping_integration_test.go | 15 +- pkg/scyllaclient/client_agent.go | 36 ++--- pkg/scyllaclient/client_agent_test.go | 123 +++++++++++----- pkg/service/cluster/service.go | 45 +++--- pkg/service/configcache/tlsconfig.go | 5 +- pkg/service/healthcheck/service.go | 6 +- .../healthcheck/service_integration_test.go | 135 +++++++++++------- .../repair/service_repair_integration_test.go | 11 +- .../service_restore_integration_test.go | 10 +- pkg/testutils/db/db.go | 4 +- pkg/testutils/exec.go | 6 + pkg/testutils/testconfig/testconfig.go | 66 +++++++++ testing/Makefile | 13 ++ testing/docker-compose.yaml | 24 ---- testing/scylla/config/scylla-ssl.yaml | 1 + 31 files changed, 501 insertions(+), 235 deletions(-) rename .github/workflows/{integration-tests-6.2.0-IPV6-tablets.yaml => integration-tests-6.2.0-IPV6-tablets-nossl.yaml} (82%) rename .github/workflows/{integration-tests-latest-enterprise-IPV4-tablets.yaml => integration-tests-latest-enterprise-IPV4-tablets-nossl.yaml} (82%) diff --git a/.github/actions/test-setup/action.yml b/.github/actions/test-setup/action.yml index bba816cc24..9ffb9bd674 100644 --- a/.github/actions/test-setup/action.yml +++ b/.github/actions/test-setup/action.yml @@ -18,6 +18,10 @@ inputs: description: "Should this action run 'make start-dev-env'" required: false default: 'true' + ssl-enabled: + description: "Specifies if Scylla cluster should use ssl only configuration or not" + required: false + default: 'true' runs: using: "composite" @@ -42,5 +46,5 @@ runs: - name: Start dev env if: inputs.start-dev-env == 'true' - run: make start-dev-env SCYLLA_VERSION=${{ inputs.scylla-version }} IP_FAMILY=${{ inputs.ip-family }} RAFT_SCHEMA=${{ inputs.raft-schema }} TABLETS=${{ inputs.tablets }} - shell: bash \ No newline at end of file + run: make start-dev-env SCYLLA_VERSION=${{ inputs.scylla-version }} IP_FAMILY=${{ inputs.ip-family }} RAFT_SCHEMA=${{ inputs.raft-schema }} TABLETS=${{ inputs.tablets }} SSL_ENABLED=${{ inputs.ssl-enabled }} + shell: bash diff --git a/.github/cfg/integration-test-cfg.yaml b/.github/cfg/integration-test-cfg.yaml index e1a34719e8..5f7e51e0fe 100644 --- a/.github/cfg/integration-test-cfg.yaml +++ b/.github/cfg/integration-test-cfg.yaml @@ -2,48 +2,58 @@ ip-family: IPV4 raft-schema: disabled tablets: none + ssl-enabled: true - scylla-version: scylla-enterprise:2023.1.11 ip-family: IPV4 raft-schema: enabled tablets: none + ssl-enabled: true - scylla-version: scylla-enterprise:2023.1.11 ip-family: IPV6 raft-schema: enabled tablets: none + ssl-enabled: true - scylla-version: scylla-enterprise:2024.1.12 ip-family: IPV4 raft-schema: none tablets: none + ssl-enabled: true - scylla-version: scylla-enterprise:2024.1.12 ip-family: IPV6 raft-schema: none tablets: none + ssl-enabled: true - scylla-version: scylla:6.2.0 ip-family: IPV4 raft-schema: none tablets: disabled + ssl-enabled: true - scylla-version: scylla:6.2.0 ip-family: IPV4 raft-schema: none tablets: enabled + ssl-enabled: true - scylla-version: scylla:6.2.0 ip-family: IPV6 raft-schema: none tablets: enabled + ssl-enabled: false - scylla-version: scylla-enterprise-nightly:latest-enterprise ip-family: IPV4 raft-schema: none tablets: disabled + ssl-enabled: true - scylla-version: scylla-enterprise-nightly:latest-enterprise ip-family: IPV4 raft-schema: none - tablets: enabled \ No newline at end of file + tablets: enabled + ssl-enabled: false diff --git a/.github/cfg/integration-test-core.yaml b/.github/cfg/integration-test-core.yaml index fc8130cd11..5a960846dd 100644 --- a/.github/cfg/integration-test-core.yaml +++ b/.github/cfg/integration-test-core.yaml @@ -27,9 +27,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} tablets: ${{ env.tablets }} + ssl-enabled: ${{ env.ssl-enabled }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' restore-schema: name: Test restore schema @@ -45,11 +46,11 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} tablets: ${{ env.tablets }} - + ssl-enabled: ${{ env.ssl-enabled }} # Go does not support negative lookahead in regex expressions, so it has to be done manually. # This regex ensures that all restore tests that didn't match restore-tables job will be run here. - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' backup: name: Test backup @@ -65,9 +66,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} tablets: ${{ env.tablets }} + ssl-enabled: ${{ env.ssl-enabled }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair @@ -83,9 +85,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} tablets: ${{ env.tablets }} + ssl-enabled: ${{ env.ssl-enabled }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair small-pkg: name: Test other, smaller packages @@ -94,9 +97,10 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup @@ -105,6 +109,7 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} tablets: ${{ env.tablets }} + ssl-enabled: ${{ env.ssl-enabled }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping @@ -128,4 +133,4 @@ jobs: run: make pkg-integration-test PKG=./pkg/store - name: Run migrate tests - run: make pkg-integration-test PKG=./pkg/schema/migrate \ No newline at end of file + run: make pkg-integration-test PKG=./pkg/schema/migrate diff --git a/.github/cfg/main.go b/.github/cfg/main.go index 7f5f15ff50..e9bcce39f4 100644 --- a/.github/cfg/main.go +++ b/.github/cfg/main.go @@ -14,6 +14,7 @@ type integrationTestCfg struct { IPFamily string `yaml:"ip-family"` RaftSchema string `yaml:"raft-schema"` Tablets string `yaml:"tablets"` + SSLEnabled string `yaml:"ssl-enabled,omitempty"` } func (cfg integrationTestCfg) name() string { @@ -29,6 +30,10 @@ func (cfg integrationTestCfg) name() string { if cfg.Tablets == "enabled" { parts = append(parts, "tablets") } + if cfg.SSLEnabled == "false" { + parts = append(parts, "nossl") + + } return strings.Join(parts, "-") } diff --git a/.github/workflows/integration-tests-2023.1.11-IPV4-raftschema.yaml b/.github/workflows/integration-tests-2023.1.11-IPV4-raftschema.yaml index d0dfb67fc5..17a9dc0bc6 100644 --- a/.github/workflows/integration-tests-2023.1.11-IPV4-raftschema.yaml +++ b/.github/workflows/integration-tests-2023.1.11-IPV4-raftschema.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV4 raft-schema: enabled tablets: none + ssl-enabled: "true" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/.github/workflows/integration-tests-2023.1.11-IPV4.yaml b/.github/workflows/integration-tests-2023.1.11-IPV4.yaml index d91e55375c..a15d1b43ee 100644 --- a/.github/workflows/integration-tests-2023.1.11-IPV4.yaml +++ b/.github/workflows/integration-tests-2023.1.11-IPV4.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV4 raft-schema: disabled tablets: none + ssl-enabled: "true" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/.github/workflows/integration-tests-2023.1.11-IPV6-raftschema.yaml b/.github/workflows/integration-tests-2023.1.11-IPV6-raftschema.yaml index cb172c7e25..1c6eb772eb 100644 --- a/.github/workflows/integration-tests-2023.1.11-IPV6-raftschema.yaml +++ b/.github/workflows/integration-tests-2023.1.11-IPV6-raftschema.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV6 raft-schema: enabled tablets: none + ssl-enabled: "true" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/.github/workflows/integration-tests-2024.1.12-IPV4.yaml b/.github/workflows/integration-tests-2024.1.12-IPV4.yaml index 325e3da867..0b6207ba2b 100644 --- a/.github/workflows/integration-tests-2024.1.12-IPV4.yaml +++ b/.github/workflows/integration-tests-2024.1.12-IPV4.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV4 raft-schema: none tablets: none + ssl-enabled: "true" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/.github/workflows/integration-tests-2024.1.12-IPV6.yaml b/.github/workflows/integration-tests-2024.1.12-IPV6.yaml index e2326ff53a..6950323ad8 100644 --- a/.github/workflows/integration-tests-2024.1.12-IPV6.yaml +++ b/.github/workflows/integration-tests-2024.1.12-IPV6.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV6 raft-schema: none tablets: none + ssl-enabled: "true" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/.github/workflows/integration-tests-6.2.0-IPV4-tablets.yaml b/.github/workflows/integration-tests-6.2.0-IPV4-tablets.yaml index 1b13543651..1a069f8f42 100644 --- a/.github/workflows/integration-tests-6.2.0-IPV4-tablets.yaml +++ b/.github/workflows/integration-tests-6.2.0-IPV4-tablets.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV4 raft-schema: none tablets: enabled + ssl-enabled: "true" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/.github/workflows/integration-tests-6.2.0-IPV4.yaml b/.github/workflows/integration-tests-6.2.0-IPV4.yaml index 38549e4c1d..11216c92b4 100644 --- a/.github/workflows/integration-tests-6.2.0-IPV4.yaml +++ b/.github/workflows/integration-tests-6.2.0-IPV4.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV4 raft-schema: none tablets: disabled + ssl-enabled: "true" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/.github/workflows/integration-tests-6.2.0-IPV6-tablets.yaml b/.github/workflows/integration-tests-6.2.0-IPV6-tablets-nossl.yaml similarity index 82% rename from .github/workflows/integration-tests-6.2.0-IPV6-tablets.yaml rename to .github/workflows/integration-tests-6.2.0-IPV6-tablets-nossl.yaml index ff0c0712c8..84b55f3839 100644 --- a/.github/workflows/integration-tests-6.2.0-IPV6-tablets.yaml +++ b/.github/workflows/integration-tests-6.2.0-IPV6-tablets-nossl.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV6 raft-schema: none tablets: enabled + ssl-enabled: "false" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping @@ -99,7 +106,7 @@ jobs: run: make pkg-integration-test PKG=./pkg/store - name: Run migrate tests run: make pkg-integration-test PKG=./pkg/schema/migrate -name: integration-tests-6.2.0-IPV6-tablets +name: integration-tests-6.2.0-IPV6-tablets-nossl "on": pull_request: types: diff --git a/.github/workflows/integration-tests-latest-enterprise-IPV4-tablets.yaml b/.github/workflows/integration-tests-latest-enterprise-IPV4-tablets-nossl.yaml similarity index 82% rename from .github/workflows/integration-tests-latest-enterprise-IPV4-tablets.yaml rename to .github/workflows/integration-tests-latest-enterprise-IPV4-tablets-nossl.yaml index 7d5316c838..b231314a78 100644 --- a/.github/workflows/integration-tests-latest-enterprise-IPV4-tablets.yaml +++ b/.github/workflows/integration-tests-latest-enterprise-IPV4-tablets-nossl.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV4 raft-schema: none tablets: enabled + ssl-enabled: "false" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping @@ -99,7 +106,7 @@ jobs: run: make pkg-integration-test PKG=./pkg/store - name: Run migrate tests run: make pkg-integration-test PKG=./pkg/schema/migrate -name: integration-tests-latest-enterprise-IPV4-tablets +name: integration-tests-latest-enterprise-IPV4-tablets-nossl "on": pull_request: types: diff --git a/.github/workflows/integration-tests-latest-enterprise-IPV4.yaml b/.github/workflows/integration-tests-latest-enterprise-IPV4.yaml index 8299cc1ddb..d86c50ddb9 100644 --- a/.github/workflows/integration-tests-latest-enterprise-IPV4.yaml +++ b/.github/workflows/integration-tests-latest-enterprise-IPV4.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV4 raft-schema: none tablets: disabled + ssl-enabled: "true" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/Makefile b/Makefile index 0029edbd0f..569ca52b3e 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,8 @@ SCYLLA_VERSION?=scylla:6.0.1 IP_FAMILY?=IPV4 RAFT_SCHEMA?=none TABLETS?=enabled +# if true starts the scylla cluster with ssl only config +SSL_ENABLED?=false MANAGER_CONFIG := testing/scylla-manager/scylla-manager.yaml PUBLIC_NET := 192.168.200. @@ -137,6 +139,7 @@ pkg-integration-test: -v "$(PWD)/testing:/integration-test/testing" \ -v "$(PWD)/$(PKG)/testdata:/integration-test/testdata" \ -w "/integration-test" \ + -e "SSL_ENABLED=$(SSL_ENABLED)" \ -u $(CURRENT_UID):$(CURRENT_GID) \ -i --read-only --rm ubuntu integration-test -test.v -test.run $(RUN) $(INTEGRATION_TEST_ARGS) $(SSL_FLAGS) $(ARGS) @@ -171,7 +174,7 @@ start-dev-env: .testing-up deploy-agent build-cli .PHONY: .testing-up .testing-up: - @IPV6=$(IPV6) SCYLLA_VERSION=$(SCYLLA_VERSION) RAFT_SCHEMA=$(RAFT_SCHEMA) TABLETS=$(TABLETS) make -C testing build down up + @IPV6=$(IPV6) SCYLLA_VERSION=$(SCYLLA_VERSION) RAFT_SCHEMA=$(RAFT_SCHEMA) TABLETS=$(TABLETS) SSL_ENABLED=$(SSL_ENABLED) make -C testing build down up .PHONY: dev-env-status dev-env-status: ## Checks status of docker containers and cluster nodes diff --git a/README.md b/README.md index 8e66c6e9b2..2ae064fa6e 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ Scylla Manager consists of tree components: |-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| | **2024.1.12** | ![integration-tests-2024.1.12-IPV4]
![integration-tests-2024.1.12-IPV6] | Restoration of schema into cluster with `consistant_cluster_management: true` is not supported | | **2023.1.11** | ![integration-tests-2023.1.11-IPV4]
![integration-tests-2023.1.11-IPV4-raftschema]
![integration-tests-2023.1.11-IPV6-raftschema] | Restoration of schema into cluster with `consistant_cluster_management: true` is not supported | -| **6.2.0** | ![integration-tests-6.2.0-IPV4]
![integration-tests-6.2.0-IPV4-tablets]
![integration-tests-6.2.0-IPV6-tablets] | Restoration of **Authentication** and **Service Levels** is not supported
Restoration of schema containing **Alternator** tables is not supported | -| **latest-enterprise** | ![integration-tests-latest-enterprise-IPV4]
![integration-tests-latest-enterprise-IPV4-tablets] | Restoration of **Authentication** and **Service Levels** is not supported
Restoration of schema containing **Alternator** tables is not supported | +| **6.2.0** | ![integration-tests-6.2.0-IPV4]
![integration-tests-6.2.0-IPV4-tablets]
![integration-tests-6.2.0-IPV6-tablets-nossl] | Restoration of **Authentication** and **Service Levels** is not supported
Restoration of schema containing **Alternator** tables is not supported | +| **latest-enterprise** | ![integration-tests-latest-enterprise-IPV4]
![integration-tests-latest-enterprise-IPV4-tablets-nossl] | Restoration of **Authentication** and **Service Levels** is not supported
Restoration of schema containing **Alternator** tables is not supported | [integration-tests-2024.1.12-IPV4]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-2024.1.12-IPV4.yaml/badge.svg?branch=branch-3.4 [integration-tests-2024.1.12-IPV6]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-2024.1.12-IPV6.yaml/badge.svg?branch=branch-3.4 @@ -29,9 +29,9 @@ Scylla Manager consists of tree components: [integration-tests-2023.1.11-IPV6-raftschema]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-2023.1.11-IPV6-raftschema.yaml/badge.svg?branch=branch-3.4 [integration-tests-6.2.0-IPV4]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-6.2.0-IPV4.yaml/badge.svg?branch=branch-3.4 [integration-tests-6.2.0-IPV4-tablets]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-6.2.0-IPV4-tablets.yaml/badge.svg?branch=branch-3.4 -[integration-tests-6.2.0-IPV6-tablets]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-6.2.0-IPV6-tablets.yaml/badge.svg?branch=branch-3.4 +[integration-tests-6.2.0-IPV6-tablets-nossl]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-6.2.0-IPV6-tablets-nossl.yaml/badge.svg?branch=branch-3.4 [integration-tests-latest-enterprise-IPV4]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-latest-enterprise-IPV4.yaml/badge.svg?branch=branch-3.4 -[integration-tests-latest-enterprise-IPV4-tablets]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-latest-enterprise-IPV4-tablets.yaml/badge.svg?branch=branch-3.4 +[integration-tests-latest-enterprise-IPV4-tablets-nossl]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-latest-enterprise-IPV4-tablets-nossl.yaml/badge.svg?branch=branch-3.4 ## Installing and updating Go @@ -81,6 +81,11 @@ for IPv6 environment: IPV6=true make start-dev-env ``` +for SSL scylla cluster: +```bash +SSL_ENABLED=true make start-dev-env +``` + This command will: 1. Build custom Scylla Docker image (testing/scylla) 2. Compile server, agent and sctool binaries diff --git a/pkg/ping/cqlping/cqlping_integration_test.go b/pkg/ping/cqlping/cqlping_integration_test.go index 7cc007bf7f..8b6bec6d4b 100644 --- a/pkg/ping/cqlping/cqlping_integration_test.go +++ b/pkg/ping/cqlping/cqlping_integration_test.go @@ -8,10 +8,11 @@ package cqlping import ( "context" "crypto/tls" - "github.com/scylladb/scylla-manager/v3/pkg/testutils/testconfig" "testing" "time" + "github.com/scylladb/scylla-manager/v3/pkg/testutils/testconfig" + "github.com/scylladb/go-log" "github.com/scylladb/scylla-manager/v3/pkg/ping" "github.com/scylladb/scylla-manager/v3/pkg/scyllaclient" @@ -24,17 +25,25 @@ func TestPingIntegration(t *testing.T) { client := newTestClient(t, log.NewDevelopmentWithLevel(zapcore.InfoLevel).Named("client"), nil) defer client.Close() - sessionHosts, err := cluster.GetRPCAddresses(context.Background(), client, []string{testconfig.ManagedClusterHost()}) + sessionHosts, err := cluster.GetRPCAddresses(context.Background(), client, []string{testconfig.ManagedClusterHost()}, !testconfig.IsSSLEnabled()) if err != nil { t.Fatal(err) } - user, password := testconfig.ManagedClusterCredentials() config := Config{ Addr: sessionHosts[0], Timeout: 250 * time.Millisecond, } + if testconfig.IsSSLEnabled() { + sslOpts := testconfig.CQLSSLOptions() + tlsConfig, err := testconfig.TLSConfig(sslOpts) + if err != nil { + t.Fatalf("setup tls config: %v", err) + } + config.TLSConfig = tlsConfig + } + t.Run("simple", func(t *testing.T) { d, err := NativeCQLPing(context.Background(), config, log.NopLogger) if err != nil { diff --git a/pkg/scyllaclient/client_agent.go b/pkg/scyllaclient/client_agent.go index 68eb87e5ab..216fe52fc8 100644 --- a/pkg/scyllaclient/client_agent.go +++ b/pkg/scyllaclient/client_agent.go @@ -70,27 +70,31 @@ func (c *Client) PinCPU(ctx context.Context, host string) error { return err } -// CQLAddr returns CQL address from NodeInfo. -// Scylla can have separate rpc_address (CQL), listen_address and respectfully -// broadcast_rpc_address and broadcast_address if some 3rd party routing -// is added. -// `fallback` argument is used in case any of above addresses is zero address. -func (ni *NodeInfo) CQLAddr(fallback string) string { - addr, port := ni.cqlAddr(fallback), ni.CQLPort() +// cqlAddr returns CQL address from NodeInfo. +func (ni *NodeInfo) cqlAddr(fallback string) string { + addr, port := ni.cqlListenAddr(fallback), ni.cqlPort() return net.JoinHostPort(addr, port) } -// CQLSSLAddr returns CQL SSL address from NodeInfo. +// cqlSSLAddr returns CQL SSL address from NodeInfo. +func (ni *NodeInfo) cqlSSLAddr(fallback string) string { + addr, port := ni.cqlListenAddr(fallback), ni.cqlSSLPort() + return net.JoinHostPort(addr, port) +} + +// CQLAddr returns either CQL or CQL SSL address from Node Info depending on the cluster configuration. // Scylla can have separate rpc_address (CQL), listen_address and respectfully // broadcast_rpc_address and broadcast_address if some 3rd party routing // is added. // `fallback` argument is used in case any of above addresses is zero address. -func (ni *NodeInfo) CQLSSLAddr(fallback string) string { - addr, port := ni.cqlAddr(fallback), ni.CQLSSLPort() - return net.JoinHostPort(addr, port) +func (ni *NodeInfo) CQLAddr(fallback string, clusterTLSAddrDisabled bool) string { + if ni.ClientEncryptionEnabled && !clusterTLSAddrDisabled { + return ni.cqlSSLAddr(fallback) + } + return ni.cqlAddr(fallback) } -func (ni *NodeInfo) cqlAddr(fallback string) string { +func (ni *NodeInfo) cqlListenAddr(fallback string) string { const ipv4Zero, ipv6Zero = "0.0.0.0", "::0" if ni.BroadcastRPCAddress != "" { @@ -109,13 +113,13 @@ func (ni *NodeInfo) cqlAddr(fallback string) string { return ni.ListenAddress } -// CQLPort returns CQL port from NodeInfo. -func (ni *NodeInfo) CQLPort() string { +// cqlPort returns CQL port from NodeInfo. +func (ni *NodeInfo) cqlPort() string { return ni.NativeTransportPort } -// CQLSSLPort returns CQL SSL port from NodeInfo. -func (ni *NodeInfo) CQLSSLPort() string { +// cqlSSLPort returns CQL SSL port from NodeInfo. +func (ni *NodeInfo) cqlSSLPort() string { return ni.NativeTransportPortSsl } diff --git a/pkg/scyllaclient/client_agent_test.go b/pkg/scyllaclient/client_agent_test.go index a51292b428..5250cc64fe 100644 --- a/pkg/scyllaclient/client_agent_test.go +++ b/pkg/scyllaclient/client_agent_test.go @@ -16,9 +16,10 @@ func TestNodeInfoCQLAddr(t *testing.T) { t.Parallel() table := []struct { - Name string - NodeInfo *scyllaclient.NodeInfo - GoldenAddress string + Name string + NodeInfo *scyllaclient.NodeInfo + ClusterDisableSSL bool + GoldenAddress string }{ { Name: "Broadcast RPC address is set", @@ -28,7 +29,8 @@ func TestNodeInfoCQLAddr(t *testing.T) { ListenAddress: "1.2.3.6", NativeTransportPort: "1234", }, - GoldenAddress: "1.2.3.4:1234", + ClusterDisableSSL: false, + GoldenAddress: "1.2.3.4:1234", }, { Name: "RPC address is set", @@ -37,7 +39,8 @@ func TestNodeInfoCQLAddr(t *testing.T) { RPCAddress: "1.2.3.5", ListenAddress: "1.2.3.6", }, - GoldenAddress: "1.2.3.5:1234", + ClusterDisableSSL: false, + GoldenAddress: "1.2.3.5:1234", }, { Name: "Listen Address is set", @@ -45,7 +48,8 @@ func TestNodeInfoCQLAddr(t *testing.T) { NativeTransportPort: "1234", ListenAddress: "1.2.3.6", }, - GoldenAddress: "1.2.3.6:1234", + ClusterDisableSSL: false, + GoldenAddress: "1.2.3.6:1234", }, { Name: "Fallback is returned when RPC Address is IPv4 zero", @@ -53,7 +57,8 @@ func TestNodeInfoCQLAddr(t *testing.T) { NativeTransportPort: "1234", RPCAddress: "0.0.0.0", }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), }, { Name: "Fallback is returned when RPC Address is IPv6 zero", @@ -61,7 +66,8 @@ func TestNodeInfoCQLAddr(t *testing.T) { NativeTransportPort: "1234", RPCAddress: "::0", }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), }, { Name: "Fallback is returned when Listen Address is IPv4 zero", @@ -69,7 +75,8 @@ func TestNodeInfoCQLAddr(t *testing.T) { NativeTransportPort: "1234", ListenAddress: "0.0.0.0", }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), }, { Name: "Fallback is returned when Listen Address is IPv6 zero", @@ -77,7 +84,8 @@ func TestNodeInfoCQLAddr(t *testing.T) { NativeTransportPort: "1234", ListenAddress: "::0", }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), }, } @@ -86,7 +94,7 @@ func TestNodeInfoCQLAddr(t *testing.T) { t.Run(test.Name, func(t *testing.T) { t.Parallel() - addr := test.NodeInfo.CQLAddr(fallback) + addr := test.NodeInfo.CQLAddr(fallback, test.ClusterDisableSSL) if addr != test.GoldenAddress { t.Errorf("expected %s address, got %s", test.GoldenAddress, addr) } @@ -98,68 +106,105 @@ func TestNodeInfoCQLSSLAddr(t *testing.T) { t.Parallel() table := []struct { - Name string - NodeInfo *scyllaclient.NodeInfo - GoldenAddress string + Name string + NodeInfo *scyllaclient.NodeInfo + ClusterDisableSSL bool + GoldenAddress string }{ { Name: "Broadcast RPC address is set", NodeInfo: &scyllaclient.NodeInfo{ - BroadcastRPCAddress: "1.2.3.4", - RPCAddress: "1.2.3.5", - ListenAddress: "1.2.3.6", - NativeTransportPortSsl: "1234", + BroadcastRPCAddress: "1.2.3.4", + RPCAddress: "1.2.3.5", + ListenAddress: "1.2.3.6", + NativeTransportPortSsl: "1234", + ClientEncryptionEnabled: true, }, - GoldenAddress: "1.2.3.4:1234", + ClusterDisableSSL: false, + GoldenAddress: "1.2.3.4:1234", }, { Name: "RPC address is set", NodeInfo: &scyllaclient.NodeInfo{ - NativeTransportPortSsl: "1234", - RPCAddress: "1.2.3.5", - ListenAddress: "1.2.3.6", + NativeTransportPortSsl: "1234", + RPCAddress: "1.2.3.5", + ListenAddress: "1.2.3.6", + ClientEncryptionEnabled: true, }, - GoldenAddress: "1.2.3.5:1234", + ClusterDisableSSL: false, + GoldenAddress: "1.2.3.5:1234", }, { Name: "Listen Address is set", NodeInfo: &scyllaclient.NodeInfo{ - NativeTransportPortSsl: "1234", - ListenAddress: "1.2.3.6", + NativeTransportPortSsl: "1234", + ListenAddress: "1.2.3.6", + ClientEncryptionEnabled: true, }, - GoldenAddress: "1.2.3.6:1234", + ClusterDisableSSL: false, + GoldenAddress: "1.2.3.6:1234", }, { Name: "Fallback is returned when RPC Address is IPv4 zero", NodeInfo: &scyllaclient.NodeInfo{ - NativeTransportPortSsl: "1234", - RPCAddress: "0.0.0.0", + NativeTransportPortSsl: "1234", + RPCAddress: "0.0.0.0", + ClientEncryptionEnabled: true, }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), }, { Name: "Fallback is returned when RPC Address is IPv6 zero", NodeInfo: &scyllaclient.NodeInfo{ - NativeTransportPortSsl: "1234", - RPCAddress: "::0", + NativeTransportPortSsl: "1234", + RPCAddress: "::0", + ClientEncryptionEnabled: true, }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), }, { Name: "Fallback is returned when Listen Address is IPv4 zero", NodeInfo: &scyllaclient.NodeInfo{ - NativeTransportPortSsl: "1234", - ListenAddress: "0.0.0.0", + NativeTransportPortSsl: "1234", + ListenAddress: "0.0.0.0", + ClientEncryptionEnabled: true, }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), }, { Name: "Fallback is returned when Listen Address is IPv6 zero", NodeInfo: &scyllaclient.NodeInfo{ - NativeTransportPortSsl: "1234", - ListenAddress: "::0", + NativeTransportPortSsl: "1234", + ListenAddress: "::0", + ClientEncryptionEnabled: true, }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), + }, + { + Name: "NativeTransportPort is returned when ssl is disabled on cluster level", + NodeInfo: &scyllaclient.NodeInfo{ + NativeTransportPort: "4321", + NativeTransportPortSsl: "1234", + ListenAddress: "1.2.3.4", + ClientEncryptionEnabled: true, + }, + ClusterDisableSSL: true, + GoldenAddress: "1.2.3.4:4321", + }, + { + Name: "NativeTransportPort is returned when Node Client Encryption is not enabled", + NodeInfo: &scyllaclient.NodeInfo{ + NativeTransportPort: "4321", + NativeTransportPortSsl: "1234", + ListenAddress: "1.2.3.4", + ClientEncryptionEnabled: false, + }, + ClusterDisableSSL: false, + GoldenAddress: "1.2.3.4:4321", }, } @@ -168,7 +213,7 @@ func TestNodeInfoCQLSSLAddr(t *testing.T) { t.Run(test.Name, func(t *testing.T) { t.Parallel() - addr := test.NodeInfo.CQLSSLAddr(fallback) + addr := test.NodeInfo.CQLAddr(fallback, test.ClusterDisableSSL) if addr != test.GoldenAddress { t.Errorf("expected %s address, got %s", test.GoldenAddress, addr) } diff --git a/pkg/service/cluster/service.go b/pkg/service/cluster/service.go index 0a9bb70778..353d67fea0 100644 --- a/pkg/service/cluster/service.go +++ b/pkg/service/cluster/service.go @@ -607,19 +607,19 @@ func (s *Service) ListNodes(ctx context.Context, clusterID uuid.UUID) ([]Node, e } // SessionConfigOption defines function modifying cluster config that can be used when creating session. -type SessionConfigOption func(ctx context.Context, clusterID uuid.UUID, client *scyllaclient.Client, cfg *gocql.ClusterConfig) error +type SessionConfigOption func(ctx context.Context, cluster *Cluster, client *scyllaclient.Client, cfg *gocql.ClusterConfig) error // SingleHostSessionConfigOption ensures that session will be connected only to the single, provided host. func SingleHostSessionConfigOption(host string) SessionConfigOption { - return func(ctx context.Context, _ uuid.UUID, client *scyllaclient.Client, cfg *gocql.ClusterConfig) error { + return func(ctx context.Context, cluster *Cluster, client *scyllaclient.Client, cfg *gocql.ClusterConfig) error { ni, err := client.NodeInfo(ctx, host) if err != nil { return errors.Wrapf(err, "fetch node (%s) info", host) } - cqlAddr := ni.CQLAddr(host) + cqlAddr := ni.CQLAddr(host, cluster.ForceTLSDisabled || cluster.ForceNonSSLSessionPort) cfg.Hosts = []string{cqlAddr} - cfg.HostFilter = gocql.WhiteListHostFilter(cqlAddr) cfg.DisableInitialHostLookup = true + cfg.HostFilter = gocql.WhiteListHostFilter(cqlAddr) return nil } } @@ -637,15 +637,21 @@ func (s *Service) GetSession(ctx context.Context, clusterID uuid.UUID, opts ...S } defer logutil.LogOnError(ctx, s.logger, client.Close, "Couldn't close scylla client") + clusterInfo, err := s.GetClusterByID(ctx, clusterID) + if err != nil { + return session, errors.Wrap(err, "cluster by id") + } + cfg := gocql.NewCluster() for _, opt := range opts { - if err := opt(ctx, clusterID, client, cfg); err != nil { + if err := opt(ctx, clusterInfo, client, cfg); err != nil { return session, err } } - // Fill hosts if they weren't specified by the options + + // Fill hosts if they weren't specified by the options or make sure that they use correct rpc address. if len(cfg.Hosts) == 0 { - sessionHosts, err := GetRPCAddresses(ctx, client, client.Config().Hosts) + sessionHosts, err := GetRPCAddresses(ctx, client, client.Config().Hosts, clusterInfo.ForceTLSDisabled || clusterInfo.ForceNonSSLSessionPort) if err != nil { s.logger.Info(ctx, "Gets session", "err", err) if errors.Is(err, ErrNoRPCAddressesFound) { @@ -662,7 +668,7 @@ func (s *Service) GetSession(ctx context.Context, clusterID uuid.UUID, opts ...S if err := s.extendClusterConfigWithAuthentication(clusterID, ni, cfg); err != nil { return session, err } - if err := s.extendClusterConfigWithTLS(ctx, clusterID, ni, cfg); err != nil { + if err := s.extendClusterConfigWithTLS(clusterInfo, ni, cfg); err != nil { return session, err } @@ -695,24 +701,15 @@ func (s *Service) extendClusterConfigWithAuthentication(clusterID uuid.UUID, ni return nil } -func (s *Service) extendClusterConfigWithTLS(ctx context.Context, clusterID uuid.UUID, ni *scyllaclient.NodeInfo, cfg *gocql.ClusterConfig) error { - cluster, err := s.GetClusterByID(ctx, clusterID) - if err != nil { - return errors.Wrap(err, "get cluster by id") - } - - cqlPort := ni.CQLPort() +func (s *Service) extendClusterConfigWithTLS(cluster *Cluster, ni *scyllaclient.NodeInfo, cfg *gocql.ClusterConfig) error { if ni.ClientEncryptionEnabled && !cluster.ForceTLSDisabled { - if !cluster.ForceNonSSLSessionPort { - cqlPort = ni.CQLSSLPort() - } cfg.SslOpts = &gocql.SslOptions{ Config: &tls.Config{ InsecureSkipVerify: true, }, } if ni.ClientEncryptionRequireAuth { - keyPair, err := s.loadTLSIdentity(clusterID) + keyPair, err := s.loadTLSIdentity(cluster.ID) if err != nil { return err } @@ -720,11 +717,6 @@ func (s *Service) extendClusterConfigWithTLS(ctx context.Context, clusterID uuid } } - p, err := strconv.Atoi(cqlPort) - if err != nil { - return errors.Wrap(err, "parse cql port") - } - cfg.Port = p return nil } @@ -770,7 +762,7 @@ var ErrNoRPCAddressesFound = errors.New("no RPC addresses found") // GetRPCAddresses accepts client and hosts parameters that are used later on to query client.NodeInfo endpoint // returning RPC addresses for given hosts. // RPC addresses are the ones that scylla uses to accept CQL connections. -func GetRPCAddresses(ctx context.Context, client *scyllaclient.Client, hosts []string) ([]string, error) { +func GetRPCAddresses(ctx context.Context, client *scyllaclient.Client, hosts []string, clusterTLSAddrDisabled bool) ([]string, error) { var sessionHosts []string var combinedError error for _, h := range hosts { @@ -779,7 +771,8 @@ func GetRPCAddresses(ctx context.Context, client *scyllaclient.Client, hosts []s combinedError = multierr.Append(combinedError, err) continue } - sessionHosts = append(sessionHosts, ni.CQLAddr(h)) + addr := ni.CQLAddr(h, clusterTLSAddrDisabled) + sessionHosts = append(sessionHosts, addr) } if len(sessionHosts) == 0 { diff --git a/pkg/service/configcache/tlsconfig.go b/pkg/service/configcache/tlsconfig.go index a0d3613389..8042405cb1 100644 --- a/pkg/service/configcache/tlsconfig.go +++ b/pkg/service/configcache/tlsconfig.go @@ -26,10 +26,7 @@ func newCQLTLSConfigIfEnabled(c *cluster.Cluster, nodeInfo *scyllaclient.NodeInf if !cqlTLSEnabled || c.ForceTLSDisabled { return nil, nil // nolint: nilnil } - cqlAddress := nodeInfo.CQLAddr(host) - if !c.ForceNonSSLSessionPort { - cqlAddress = nodeInfo.CQLSSLAddr(host) - } + cqlAddress := nodeInfo.CQLAddr(host, c.ForceTLSDisabled || c.ForceNonSSLSessionPort) tlsConfig := &tls.Config{ InsecureSkipVerify: true, } diff --git a/pkg/service/healthcheck/service.go b/pkg/service/healthcheck/service.go index 1213682963..8407e0e281 100644 --- a/pkg/service/healthcheck/service.go +++ b/pkg/service/healthcheck/service.go @@ -331,9 +331,13 @@ func (s *Service) decorateNodeStatus(status *NodeStatus, ni configcache.NodeConf } func (s *Service) pingCQL(ctx context.Context, clusterID uuid.UUID, host string, timeout time.Duration, ni configcache.NodeConfig) (rtt time.Duration, err error) { + cluster, err := s.clusterProvider(ctx, clusterID) + if err != nil { + return rtt, errors.Wrap(err, "cluster provider") + } // Try to connect directly to host address. config := cqlping.Config{ - Addr: ni.CQLAddr(host), + Addr: ni.CQLAddr(host, cluster.ForceTLSDisabled || cluster.ForceNonSSLSessionPort), Timeout: timeout, } diff --git a/pkg/service/healthcheck/service_integration_test.go b/pkg/service/healthcheck/service_integration_test.go index 7dbe0b4376..4744e24148 100644 --- a/pkg/service/healthcheck/service_integration_test.go +++ b/pkg/service/healthcheck/service_integration_test.go @@ -11,6 +11,7 @@ import ( "io" "net" "net/http" + "os" "strings" "testing" "time" @@ -77,6 +78,8 @@ func TestStatus_Ping_Independent_From_REST_Integration(t *testing.T) { Host: hostWithUnresponsiveREST, AuthToken: "token", } + clusterWithSSL(t, testCluster, IsSSLEnabled()) + err = clusterSvc.PutCluster(context.Background(), testCluster) if err != nil { t.Fatal(err) @@ -161,12 +164,14 @@ func TestStatusIntegration(t *testing.T) { Host: "192.168.200.11", AuthToken: "token", } + clusterWithSSL(t, c, IsSSLEnabled()) + err = clusterSvc.PutCluster(context.Background(), c) if err != nil { t.Fatal(err) } - testStatusIntegration(t, c.ID, clusterSvc, clusterSvc.GetClusterByID, s) + testStatusIntegration(t, c.ID, clusterSvc, clusterSvc.GetClusterByID, s, IsSSLEnabled()) } func TestStatusWithCQLCredentialsIntegration(t *testing.T) { @@ -190,15 +195,17 @@ func TestStatusWithCQLCredentialsIntegration(t *testing.T) { Username: username, Password: password, } + clusterWithSSL(t, c, IsSSLEnabled()) + err = clusterSvc.PutCluster(context.Background(), c) if err != nil { t.Fatal(err) } - testStatusIntegration(t, c.ID, clusterSvc, clusterSvc.GetClusterByID, s) + testStatusIntegration(t, c.ID, clusterSvc, clusterSvc.GetClusterByID, s, IsSSLEnabled()) } -func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster.Servicer, clusterProvider cluster.ProviderFunc, secretsStore store.Store) { +func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster.Servicer, clusterProvider cluster.ProviderFunc, secretsStore store.Store, sslEnabled bool) { logger := log.NewDevelopmentWithLevel(zapcore.InfoLevel).Named("healthcheck") // Tests here do not test the dynamic t/o functionality @@ -264,12 +271,12 @@ func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster } golden := []NodeStatus{ - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, } assertEqual(t, golden, status) }) @@ -290,20 +297,20 @@ func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster } golden := []NodeStatus{ - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "TIMEOUT", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "TIMEOUT", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, } assertEqual(t, golden, status) }) t.Run("node CQL TIMEOUT", func(t *testing.T) { host := IPFromTestNet("12") - blockCQL(t, host) - defer unblockCQL(t, host) + blockCQL(t, host, sslEnabled) + defer unblockCQL(t, host, sslEnabled) status, err := s.Status(context.Background(), clusterID) if err != nil { @@ -316,12 +323,12 @@ func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster } golden := []NodeStatus{ - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "TIMEOUT", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "TIMEOUT", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, } assertEqual(t, golden, status) }) @@ -342,12 +349,12 @@ func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster } golden := []NodeStatus{ - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "TIMEOUT"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "TIMEOUT", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, } assertEqual(t, golden, status) }) @@ -368,12 +375,12 @@ func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster } golden := []NodeStatus{ - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "DOWN", RESTCause: "dial tcp " + URLEncodeIP(ToCanonicalIP(IPFromTestNet("12"))) + ":10001: connect: connection refused", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "DOWN", RESTCause: "dial tcp " + URLEncodeIP(ToCanonicalIP(IPFromTestNet("12"))) + ":10001: connect: connection refused", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, } assertEqual(t, golden, status) }) @@ -393,12 +400,12 @@ func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster } golden := []NodeStatus{ - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "UNAUTHORIZED", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "UNAUTHORIZED", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, } assertEqual(t, golden, status) }) @@ -418,12 +425,12 @@ func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster } golden := []NodeStatus{ - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "HTTP 502", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "HTTP 502", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, } assertEqual(t, golden, status) }) @@ -482,16 +489,24 @@ func tryUnblockREST(t *testing.T, hosts []string) { } } -func blockCQL(t *testing.T, h string) { +func blockCQL(t *testing.T, h string, sslEnabled bool) { t.Helper() - if err := RunIptablesCommand(h, CmdBlockScyllaCQL); err != nil { + cmd := CmdBlockScyllaCQL + if sslEnabled { + cmd = CmdBlockScyllaCQLSSL + } + if err := RunIptablesCommand(h, cmd); err != nil { t.Error(err) } } -func unblockCQL(t *testing.T, h string) { +func unblockCQL(t *testing.T, h string, sslEnabled bool) { t.Helper() - if err := RunIptablesCommand(h, CmdUnblockScyllaCQL); err != nil { + cmd := CmdUnblockScyllaCQL + if sslEnabled { + cmd = CmdUnblockScyllaCQLSSL + } + if err := RunIptablesCommand(h, cmd); err != nil { t.Error(err) } } @@ -577,3 +592,21 @@ func fakeHealthCheckStatus(host string, code int) http.RoundTripper { return nil, nil }) } + +func clusterWithSSL(t *testing.T, cluster *cluster.Cluster, sslEnabled bool) { + t.Helper() + if !sslEnabled { + return + } + sslOpts := CQLSSLOptions() + userKey, err := os.ReadFile(sslOpts.KeyPath) + if err != nil { + t.Fatalf("read file (%s) err: %v", sslOpts.KeyPath, err) + } + userCrt, err := os.ReadFile(sslOpts.CertPath) + if err != nil { + t.Fatalf("read file (%s) err: %v", sslOpts.CertPath, err) + } + cluster.SSLUserKeyFile = userKey + cluster.SSLUserCertFile = userCrt +} diff --git a/pkg/service/repair/service_repair_integration_test.go b/pkg/service/repair/service_repair_integration_test.go index f2bc2aedfe..6c8d5c5493 100644 --- a/pkg/service/repair/service_repair_integration_test.go +++ b/pkg/service/repair/service_repair_integration_test.go @@ -33,6 +33,7 @@ import ( "github.com/scylladb/scylla-manager/v3/pkg/schema/table" "github.com/scylladb/scylla-manager/v3/pkg/service/cluster" "github.com/scylladb/scylla-manager/v3/pkg/service/scheduler" + "github.com/scylladb/scylla-manager/v3/pkg/testutils/testconfig" . "github.com/scylladb/scylla-manager/v3/pkg/testutils/testconfig" . "github.com/scylladb/scylla-manager/v3/pkg/testutils/testhelper" "github.com/scylladb/scylla-manager/v3/pkg/util" @@ -1231,9 +1232,17 @@ func TestServiceRepairIntegration(t *testing.T) { } cfg := cqlping.Config{ - Addr: ni.CQLAddr(ignored), + Addr: ni.CQLAddr(ignored, false), Timeout: time.Minute, } + if testconfig.IsSSLEnabled() { + sslOpts := testconfig.CQLSSLOptions() + tlsConfig, err := testconfig.TLSConfig(sslOpts) + if err != nil { + t.Fatalf("setup tls config: %v", err) + } + cfg.TLSConfig = tlsConfig + } cond := func() bool { if _, err = cqlping.QueryPing(ctx, cfg, TestDBUsername(), TestDBPassword()); err != nil { diff --git a/pkg/service/restore/service_restore_integration_test.go b/pkg/service/restore/service_restore_integration_test.go index d3a79ff39f..810edbe934 100644 --- a/pkg/service/restore/service_restore_integration_test.go +++ b/pkg/service/restore/service_restore_integration_test.go @@ -30,6 +30,7 @@ import ( "github.com/scylladb/scylla-manager/v3/pkg/service/backup" "github.com/scylladb/scylla-manager/v3/pkg/service/repair" . "github.com/scylladb/scylla-manager/v3/pkg/service/restore" + "github.com/scylladb/scylla-manager/v3/pkg/testutils/testconfig" . "github.com/scylladb/scylla-manager/v3/pkg/testutils/testhelper" "github.com/scylladb/scylla-manager/v3/pkg/util/jsonutil" "go.uber.org/atomic" @@ -1737,13 +1738,20 @@ func (h *restoreTestHelper) restartScylla() { b := backoff.WithContext(backoff.WithMaxRetries( backoff.NewConstantBackOff(500*time.Millisecond), 10), ctx) if err := backoff.Retry(func() error { - sessionHosts, err = cluster.GetRPCAddresses(ctx, h.Client, []string{host}) + sessionHosts, err = cluster.GetRPCAddresses(ctx, h.Client, []string{host}, false) return err }, b); err != nil { h.T.Fatal(err) } cfg.Addr = sessionHosts[0] + if testconfig.IsSSLEnabled() { + sslOpts := testconfig.CQLSSLOptions() + cfg.TLSConfig, err = testconfig.TLSConfig(sslOpts) + if err != nil { + h.T.Fatalf("tls config: %v", err) + } + } cond := func() bool { if _, err = cqlping.QueryPing(ctx, cfg, TestDBUsername(), TestDBPassword()); err != nil { return false diff --git a/pkg/testutils/db/db.go b/pkg/testutils/db/db.go index 5d59b6ac5d..598470e25d 100644 --- a/pkg/testutils/db/db.go +++ b/pkg/testutils/db/db.go @@ -83,7 +83,7 @@ func CreateManagedClusterSession(tb testing.TB, empty bool, client *scyllaclient tb.Helper() ctx := context.Background() - sessionHosts, err := cluster.GetRPCAddresses(ctx, client, client.Config().Hosts) + sessionHosts, err := cluster.GetRPCAddresses(ctx, client, client.Config().Hosts, false) if err != nil { tb.Log(err) if errors.Is(err, cluster.ErrNoRPCAddressesFound) { @@ -101,7 +101,7 @@ func CreateManagedClusterSession(tb testing.TB, empty bool, client *scyllaclient Username: user, Password: pass, } - if os.Getenv("SSL_ENABLED") != "" { + if os.Getenv("SSL_ENABLED") == "true" { cluster.SslOpts = testconfig.CQLSSLOptions() cluster.Port = testconfig.CQLPort() } diff --git a/pkg/testutils/exec.go b/pkg/testutils/exec.go index 0b768a2f4e..624dea384f 100644 --- a/pkg/testutils/exec.go +++ b/pkg/testutils/exec.go @@ -24,6 +24,12 @@ const ( // CmdUnblockScyllaCQL defines the command used for unblocking the Scylla CQL access. CmdUnblockScyllaCQL = "iptables -D INPUT -p tcp --destination-port 9042 -j DROP" + // CmdBlockScyllaCQLSSL defines the command used for blocking the Scylla CQL access. + CmdBlockScyllaCQLSSL = "iptables -A INPUT -p tcp --destination-port 9142 -j DROP" + + // CmdUnblockScyllaCQLSSL defines the command used for unblocking the Scylla CQL access. + CmdUnblockScyllaCQLSSL = "iptables -D INPUT -p tcp --destination-port 9142 -j DROP" + // CmdBlockScyllaAlternator defines the command used for blocking the Scylla Alternator access. CmdBlockScyllaAlternator = "iptables -A INPUT -p tcp --destination-port 8000 -j DROP" diff --git a/pkg/testutils/testconfig/testconfig.go b/pkg/testutils/testconfig/testconfig.go index 2ac2231c51..040218fdd5 100644 --- a/pkg/testutils/testconfig/testconfig.go +++ b/pkg/testutils/testconfig/testconfig.go @@ -3,7 +3,13 @@ package testconfig import ( + "crypto/tls" + "crypto/x509" + "errors" "flag" + "fmt" + "os" + "strconv" "strings" "time" @@ -123,3 +129,63 @@ func ScyllaManagerDBCluster() string { } return *flagCluster } + +// IsSSLEnabled is a helper function to parse SSL_ENABLED env var. +// SSL_ENABLED env var indicates if scylla cluster is configured to use ssl or not. +func IsSSLEnabled() bool { + sslEnabled, err := strconv.ParseBool(os.Getenv("SSL_ENABLED")) + if err != nil { + panic("parse SSL_ENABLED env var:" + err.Error()) + } + return sslEnabled +} + +// TLSConfig returns tls.Config to work ssl enabled scylla cluster. +// this function is almost an exact copy of setupTLSConfig from github.com/gocql/gocql/connectionpool.go. +func TLSConfig(sslOpts *gocql.SslOptions) (*tls.Config, error) { + // Config.InsecureSkipVerify | EnableHostVerification | Result + // Config is nil | true | verify host + // Config is nil | false | do not verify host + // false | false | verify host + // true | false | do not verify host + // false | true | verify host + // true | true | verify host + var tlsConfig *tls.Config + if sslOpts.Config == nil { + tlsConfig = &tls.Config{ + InsecureSkipVerify: !sslOpts.EnableHostVerification, + } + } else { + // use clone to avoid race. + tlsConfig = sslOpts.Config.Clone() + } + + if tlsConfig.InsecureSkipVerify && sslOpts.EnableHostVerification { + tlsConfig.InsecureSkipVerify = false + } + // ca cert is optional. + if sslOpts.CaPath != "" { + if tlsConfig.RootCAs == nil { + tlsConfig.RootCAs = x509.NewCertPool() + } + + pem, err := os.ReadFile(sslOpts.CaPath) + if err != nil { + return nil, fmt.Errorf("connectionpool: unable to open CA certs: %w", err) + } + + if !tlsConfig.RootCAs.AppendCertsFromPEM(pem) { + return nil, errors.New("connectionpool: failed parsing or CA certs") + } + } + + if sslOpts.CertPath != "" || sslOpts.KeyPath != "" { + mycert, err := tls.LoadX509KeyPair(sslOpts.CertPath, sslOpts.KeyPath) + if err != nil { + return nil, fmt.Errorf("connectionpool: unable to load X509 key pair: %w", err) + } + tlsConfig.Certificates = append(tlsConfig.Certificates, mycert) + } + + return tlsConfig, nil +} diff --git a/testing/Makefile b/testing/Makefile index 3d9b888f3e..fadbf85582 100644 --- a/testing/Makefile +++ b/testing/Makefile @@ -63,7 +63,17 @@ up: @cd scylla/certs && ./generate.sh @echo "==> Generating Scylla configuration" @cp scylla/config/scylla.yaml scylla/scylla.yaml + +ifeq ($(SSL_ENABLED),true) + # disable non-ssl port + @$(YQ) delete -i scylla/scylla.yaml 'native_transport_port' + # merge into scylla.yaml values from config/scylla-ssl.yaml with overwrite option (-x) + @$(YQ) merge -i -x scylla/scylla.yaml scylla/config/scylla-ssl.yaml + @cp scylla/config/cqlshrc-ssl scylla/cqlshrc +else @cp scylla/config/cqlshrc scylla/cqlshrc +endif + ifeq ($(RAFT_SCHEMA),enabled) @$(YQ) write -i scylla/scylla.yaml 'consistent_cluster_management' true endif @@ -108,6 +118,9 @@ endif @until [ 1 -le $$($(SM_NODETOOL) status | grep -c "UN") ]; do echo -n "."; sleep 2; done ; echo "" @./nodes_exec "rm /root/.cqlshrc || true" + @./nodes_exec "mkdir -p /root/.cassandra" + @./nodes_cp "scylla/cqlshrc" "/root/.cassandra/cqlshrc" + @echo "==> Adding Minio user" ./minio/add_user.sh || true @echo "==> Initialising cluster" diff --git a/testing/docker-compose.yaml b/testing/docker-compose.yaml index 03acfd961f..d4c9530c07 100644 --- a/testing/docker-compose.yaml +++ b/testing/docker-compose.yaml @@ -12,9 +12,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: @@ -32,9 +29,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: @@ -52,9 +46,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs/ - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: @@ -72,9 +63,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: @@ -92,9 +80,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: @@ -112,9 +97,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: @@ -132,9 +114,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: @@ -152,9 +131,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: diff --git a/testing/scylla/config/scylla-ssl.yaml b/testing/scylla/config/scylla-ssl.yaml index b151d1a330..8ee09b0af2 100644 --- a/testing/scylla/config/scylla-ssl.yaml +++ b/testing/scylla/config/scylla-ssl.yaml @@ -1,4 +1,5 @@ # Scylla SSL enabled configuration. +# This file is merged with testing/config/scylla.yaml during start-dev-env process when SSL_ENABLED=true. native_transport_port_ssl: 9142 client_encryption_options: