Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[2.12] excluding delete system index tests from a security enabled cluster #222

Merged
merged 5 commits into from
Feb 9, 2024
Merged

[2.12] excluding delete system index tests from a security enabled cluster #222

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f09d81c
excluding delete system index tests from a security enabled cluster
joshpalis Feb 9, 2024
bf608e1
moving security plugin configuration down to integ test, only downloa…
joshpalis Feb 9, 2024
43ab424
Adding security test workflow to CI
joshpalis Feb 9, 2024
fd420cb
adding comment on use of admin password
joshpalis Feb 9, 2024
1cab759
Changing security workflow name
joshpalis Feb 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions .github/workflows/test_security.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
name: Run Security tests
on:
push:
branches-ignore:
- 'whitesource-remediate/**'
- 'backport/**'
pull_request:
types: [opened, synchronize, reopened]

jobs:
Get-CI-Image-Tag:
uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
with:
product: opensearch

integ-test-with-security-linux:
strategy:
matrix:
java: [11, 17, 21]

name: Run Security Integration Tests on Linux
runs-on: ubuntu-latest
needs: Get-CI-Image-Tag
container:
# using the same image which is used by opensearch-build team to build the OpenSearch Distribution
# this image tag is subject to change as more dependencies and updates will arrive over time
image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
# need to switch to root so that github actions can install runner binary on container without permission issues.
options: --user root

steps:
- name: Checkout Skills
uses: actions/checkout@v3
- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: ${{ matrix.java }}
- name: Run tests
# switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip.
run: |
chown -R 1000:1000 `pwd`
su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true"
180 changes: 108 additions & 72 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,79 +251,8 @@ opensearch_tmp_dir.mkdirs()

ext {
projectSubstitutions = [:]

// Config below including files are copied from security demo configuration
['esnode.pem', 'esnode-key.pem', 'root-ca.pem','kirk.pem','kirk-key.pem'].forEach { file ->
File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile()
download.run {
src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file
dest local
overwrite false
}
}

isSnapshot = "true" == System.getProperty("build.snapshot", "true")
projectSubstitutions = [:]

configureSecurityPlugin = { OpenSearchCluster cluster ->
configurations.secureIntegTestPluginArchive.asFileTree.each {
if(it.name.contains("opensearch-security")){
cluster.plugin(provider(new Callable<RegularFile>() {
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
return it
}
}
}
}))
}
}

cluster.getNodes().forEach { node ->
var creds = node.getCredentials()
if (creds.isEmpty()) {
creds.add(Map.of('username', 'admin', 'password', 'admin'))
} else {
creds.get(0).putAll(Map.of('username', 'admin', 'password', 'admin'))
}
}

// // Config below including files are copied from security demo configuration
cluster.extraConfigFile("esnode.pem", file("$opensearch_tmp_dir/esnode.pem"))
cluster.extraConfigFile("esnode-key.pem", file("$opensearch_tmp_dir/esnode-key.pem"))
cluster.extraConfigFile("root-ca.pem", file("$opensearch_tmp_dir/root-ca.pem"))

// This configuration is copied from the security plugins demo install:
// https://github.com/opensearch-project/security/blob/2.11.1.0/tools/install_demo_configuration.sh#L365-L388
cluster.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
cluster.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
cluster.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
cluster.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
cluster.setting("plugins.security.ssl.http.enabled", "true")
cluster.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
cluster.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
cluster.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
cluster.setting("plugins.security.allow_unsafe_democertificates", "true")
cluster.setting("plugins.security.allow_default_init_securityindex", "true")
cluster.setting("plugins.security.unsupported.inject_user.enabled", "true")

cluster.setting("plugins.security.authcz.admin_dn", "\n- CN=kirk,OU=client,O=client,L=test, C=de")
cluster.setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]')
cluster.setting('plugins.security.system_indices.enabled', "true")
cluster.setting('plugins.security.system_indices.indices', '[' +
'".plugins-ml-config", ' +
'".plugins-ml-connector", ' +
'".plugins-ml-model-group", ' +
'".plugins-ml-model", ".plugins-ml-task", ' +
'".plugins-ml-conversation-meta", ' +
'".plugins-ml-conversation-interactions", ' +
']'
)
cluster.setSecure(true)
}
}

allprojects {
Expand Down Expand Up @@ -426,15 +355,28 @@ integTest {

if (System.getProperty("security.enabled") != null) {
// If security is enabled, set is_https/user/password defaults
// admin password is permissable here since the security plugin is manually configured using the default internal_users.yml configuration
is_https = is_https == null ? "true" : is_https
user = user == null ? "admin" : user
password = password == null ? "admin" : password
joshpalis marked this conversation as resolved.
Show resolved Hide resolved
System.setProperty("https", is_https)
System.setProperty("user", user)
System.setProperty("password", password)
}

systemProperty("https", is_https)
systemProperty("user", user)
systemProperty("password", password)

if (System.getProperty("https") != null && System.getProperty("https") == "true") {
filter {
excludeTestsMatching "org.opensearch.integTest.SearchAlertsToolIT"
excludeTestsMatching "org.opensearch.integTest.SearchAnomalyDetectorsToolIT"
excludeTestsMatching "org.opensearch.integTest.SearchAnomalyResultsToolIT"
excludeTestsMatching "org.opensearch.integTest.SearchMonitorsToolIT"
}
}


// doFirst delays this block until execution time
doFirst {
Expand Down Expand Up @@ -463,7 +405,92 @@ testClusters.integTest {

// Optionally install security
if (System.getProperty("security.enabled") != null) {
configureSecurityPlugin(testClusters.integTest)
configurations.secureIntegTestPluginArchive.asFileTree.each {
if(it.name.contains("opensearch-security")){
plugin(provider(new Callable<RegularFile>() {
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
return it
}
}
}
}))
}
}

getNodes().forEach { node ->
var creds = node.getCredentials()
// admin password is permissable here since the security plugin is manually configured using the default internal_users.yml configuration
if (creds.isEmpty()) {
creds.add(Map.of('username', 'admin', 'password', 'admin'))
joshpalis marked this conversation as resolved.
Show resolved Hide resolved
} else {
creds.get(0).putAll(Map.of('username', 'admin', 'password', 'admin'))
}
}

// Config below including files are copied from security demo configuration
['esnode.pem', 'esnode-key.pem', 'root-ca.pem','kirk.pem','kirk-key.pem'].forEach { file ->
File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile()
download.run {
src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file
dest local
overwrite false
}
}

// // Config below including files are copied from security demo configuration
extraConfigFile("esnode.pem", file("$opensearch_tmp_dir/esnode.pem"))
extraConfigFile("esnode-key.pem", file("$opensearch_tmp_dir/esnode-key.pem"))
extraConfigFile("root-ca.pem", file("$opensearch_tmp_dir/root-ca.pem"))

// This configuration is copied from the security plugins demo install:
// https://github.com/opensearch-project/security/blob/2.11.1.0/tools/install_demo_configuration.sh#L365-L388
setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
setting("plugins.security.ssl.http.enabled", "true")
setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
setting("plugins.security.allow_unsafe_democertificates", "true")
setting("plugins.security.allow_default_init_securityindex", "true")
setting("plugins.security.unsupported.inject_user.enabled", "true")

setting("plugins.security.authcz.admin_dn", "\n- CN=kirk,OU=client,O=client,L=test, C=de")
setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]')
setting('plugins.security.system_indices.enabled', "true")
setting('plugins.security.system_indices.indices', '[' +
'".plugins-ml-config", ' +
'".plugins-ml-connector", ' +
'".plugins-ml-model-group", ' +
'".plugins-ml-model", ".plugins-ml-task", ' +
'".plugins-ml-conversation-meta", ' +
'".plugins-ml-conversation-interactions", ' +
'".opendistro-alerting-config", ' +
'".opendistro-alerting-alert*", ' +
'".opendistro-anomaly-results*", ' +
'".opendistro-anomaly-detector*", ' +
'".opendistro-anomaly-checkpoints", ' +
'".opendistro-anomaly-detection-state", ' +
'".opendistro-reports-*", ' +
'".opensearch-notifications-*", ' +
'".opensearch-notebooks", ' +
'".opensearch-observability", ' +
'".ql-datasources", ' +
'".opendistro-asynchronous-search-response*", ' +
'".replication-metadata-store", ' +
'".opensearch-knn-models", ' +
'".geospatial-ip2geo-data*", ' +
'".plugins-flow-framework-config", ' +
'".plugins-flow-framework-templates", ' +
'".plugins-flow-framework-state"' +
']'
)
setSecure(true)
}

// Installs all registered zipArchive dependencies on integTest cluster nodes
Expand Down Expand Up @@ -517,6 +544,15 @@ task integTestRemote(type: RestIntegTestTask) {
includeTestsMatching "org.opensearch.integTest.*IT"
}
}

if (System.getProperty("https") != null && System.getProperty("https") == "true") {
filter {
excludeTestsMatching "org.opensearch.integTest.SearchAlertsToolIT"
excludeTestsMatching "org.opensearch.integTest.SearchAnomalyDetectorsToolIT"
excludeTestsMatching "org.opensearch.integTest.SearchAnomalyResultsToolIT"
excludeTestsMatching "org.opensearch.integTest.SearchMonitorsToolIT"
}
}
}

// Automatically sets up the integration test cluster locally
Expand Down
Loading