Merge branch 'main' into remove-openssl

.github/actions/start-opensearch-with-one-plugin/action.yml

@@ -1,4 +1,4 @@
-name: 'Start OpenSearch with One Plugin'
+name: 'Launch OpenSearch with a single plugin installed'
 description: 'Downloads latest build of OpenSearch, installs a plugin, executes a script and then starts OpenSearch on localhost:9200'
 
 inputs:
@@ -10,14 +10,10 @@ inputs:
     description: 'The name of the plugin to use, such as opensearch-security'
     required: true
 
-  plugin-start-script:
-    description: 'The file name for the configuration script for the plugin such as install_demo_configurations -- may not be needed for every plugin'
+  setup-script-name:
+    description: 'The name of the setup script you want to run i.e. "setup" (do not include file extension). Leave empty to indicate one should not be run.'
     required: false
 
-  docker-host-plugin-zip:
-    description: 'The name of the zip file for the plugin hosted on docker-host i.e. security-plugin.zip '
-    required: true
-
 runs:
   using: "composite"
   steps:
@@ -29,71 +25,66 @@ runs:
       shell: pwsh
 
     # Download OpenSearch
-    - uses: peternied/download-file@v1
+    - name: Download OpenSearch for Windows
+      uses: peternied/download-file@v2
       if: ${{ runner.os == 'Windows' }}
       with:
         url: https://artifacts.opensearch.org/snapshots/core/opensearch/${{ inputs.opensearch-version }}-SNAPSHOT/opensearch-min-${{ inputs.opensearch-version }}-SNAPSHOT-windows-x64-latest.zip
 
-    - uses: peternied/download-file@v1
+
+    - name: Download OpenSearch for Linux
+      uses: peternied/download-file@v2
       if: ${{ runner.os == 'Linux' }}
       with:
-        url: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${{ inputs.opensearch-version }}/latest/linux/x64/tar/builds/opensearch/dist/opensearch-min-${{ inputs.opensearch-version }}-linux-x64.tar.gz
+        url: https://artifacts.opensearch.org/snapshots/core/opensearch/${{ inputs.opensearch-version }}-SNAPSHOT/opensearch-min-${{ inputs.opensearch-version }}-SNAPSHOT-linux-x64-latest.tar.gz
 
     # Extract downloaded zip
-    - name: Extract downloaded zip for Linux
+    - name: Extract downloaded tar
       if: ${{ runner.os == 'Linux' }}
       run: |
         tar -xzf opensearch-*.tar.gz
         rm -f opensearch-*.tar.gz
       shell: bash
 
-    - name: Extract downloaded zip for Windows
+    - name: Extract downloaded zip
       if: ${{ runner.os == 'Windows' }}
       run: |
         tar -xzf opensearch-min-${{ inputs.opensearch-version }}-SNAPSHOT-windows-x64-latest.zip
         del opensearch-min-${{ inputs.opensearch-version }}-SNAPSHOT-windows-x64-latest.zip
       shell: pwsh
 
-    # Move and rename the plugin for installation
-    - name: Move and rename the plugin for installation
-      run: mv ./build/distributions/${{ inputs.plugin-name }}-*.zip ${{ inputs.plugin-name }}.zip
-      shell: bash
-
-    # Install the plugin, runs its start-script, and start the OpenSearch server
+    # Install the plugin
     - name: Install Plugin into OpenSearch for Linux
       if: ${{ runner.os == 'Linux'}}
       run: |
-        cat > os-ep.sh <<EOF
-        yes | opensearch-plugin install file:///docker-host/${{ inputs.docker-host-plugin-zip }}.zip
-        chmod +x plugins/${{ inputs.plugin-name }}/tools/${{ inputs.plugin-start-script }}.sh
-        yes | plugins/${{ inputs.plugin-name }}/tools/${{ inputs.plugin-start-script }}.sh
-        chown 1001:1001 -R /opensearch
-        su -c "/opensearch/bin/opensearch" -s /bin/bash opensearch
-        EOF
-        docker build -t opensearch-test -f- . <<EOF
-        FROM ubuntu:latest
-        COPY --chown=1001:1001 os-ep.sh /docker-host/
-        COPY --chown=1001:1001 ${{ inputs.plugin-name }}.zip /docker-host/${{ inputs.docker-host-plugin-zip }}.zip
-        COPY --chown=1001:1001 opensearch* /opensearch/
-        RUN chmod +x /docker-host/os-ep.sh
-        RUN useradd -u 1001 -s /sbin/nologin opensearch
-        ENV PATH="/opensearch/bin:${PATH}"
-        WORKDIR /opensearch/
-        ENTRYPOINT /docker-host/os-ep.sh
-        EOF
+        chmod +x ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT/bin/opensearch-plugin
+        /bin/bash -c "yes | ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT/bin/opensearch-plugin install file:$(pwd)/opensearch-security.zip"
       shell: bash
 
     - name: Install Plugin into OpenSearch for Windows
       if: ${{ runner.os == 'Windows'}}
       run: |
-        'y' | .\opensearch-${{ inputs.opensearch-version }}-SNAPSHOT\bin\opensearch-plugin.bat install file:$pwd\${{ inputs.plugin-name }}.zip
-        'y', 'y', 'N' | .\opensearch-${{ inputs.opensearch-version }}-SNAPSHOT\plugins\${{ inputs.plugin-name }}\tools\${{ inputs.plugin-start-script }}.bat
+        'y' | .\opensearch-${{ inputs.opensearch-version }}-SNAPSHOT\bin\opensearch-plugin.bat install file:$(pwd)\${{ inputs.plugin-name }}.zip
+      shell: pwsh
+
+    # Run any configuration scripts
+    - name: Run Setup Script for Linux
+      if: ${{ runner.os == 'Linux' && inputs.setup-script-name != '' }}
+      run: | 
+        echo "running linux setup"
+        chmod +x ./${{ inputs.setup-script-name }}.sh
+        ./${{ inputs.setup-script-name }}.sh
+      shell: bash
+
+    - name: Run Setup Script for Windows
+      if: ${{ runner.os == 'Windows' && inputs.setup-script-name != '' }}
+      run: .\${{ inputs.setup-script-name }}.bat
       shell: pwsh
 
     # Run OpenSearch
     - name: Run OpenSearch with plugin on Linux
       if: ${{ runner.os == 'Linux'}}
-      run: docker run --name ops -d -p 9200:9200 -p 9600:9600 -i opensearch-test:latest
+      run: /bin/bash -c "./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT/bin/opensearch &"
       shell: bash
 
     - name: Run OpenSearch with plugin on Windows
@@ -110,9 +101,7 @@ runs:
     # Verify that the server is operational
     - name: Check OpenSearch Running on Linux
       if: ${{ runner.os != 'Windows'}}
-      run: |
-        docker logs ops
-        curl https://localhost:9200/_cat/plugins -u 'admin:admin' -k -v
+      run: curl https://localhost:9200/_cat/plugins -u 'admin:admin' -k -v
       shell: bash
 
     - name: Check OpenSearch Running on Windows

.github/workflows/integration-tests.yml

@@ -1,6 +1,6 @@
 name: Bulk Integration Test
 
-on: [ workflow_dispatch ]
+on: [workflow_dispatch]
 
 env:
   GRADLE_OPTS: -Dhttp.keepAlive=false

.github/workflows/plugin_install.yml

@@ -2,14 +2,19 @@ name: Plugin Install
 
 on: [push, pull_request, workflow_dispatch]
 
+env:
+  OPENSEARCH_VERSION: 3.0.0
+  PLUGIN_NAME: opensearch-security
+
 jobs:
   plugin-install:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-latest, ubuntu-latest]
+        os: [ubuntu-latest, windows-latest]
         jdk: [11, 17]
     runs-on: ${{ matrix.os }}
+
     steps:
       - name: Set up JDK
         uses: actions/setup-java@v1
@@ -20,15 +25,36 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Assemble target plugin
-        run: ./gradlew assemble
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: assemble
+
+      # Move and rename the plugin for installation
+      - name: Move and rename the plugin for installation
+        run: mv ./build/distributions/${{ env.PLUGIN_NAME }}-*.zip ${{ env.PLUGIN_NAME }}.zip
+        shell: bash
+
+      - name: Create Setup Script
+        if: ${{ runner.os == 'Linux' }}
+        run: |
+          cat > setup.sh <<'EOF'
+          chmod +x  ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/plugins/${{ env.PLUGIN_NAME }}/tools/install_demo_configuration.sh 
+          /bin/bash -c "yes | ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/plugins/${{ env.PLUGIN_NAME }}/tools/install_demo_configuration.sh"
+          EOF
+
+      - name: Create Setup Script
+        if: ${{ runner.os == 'Windows' }}
+        run: |
+          New-Item .\setup.bat -type file
+          Set-Content .\setup.bat -Value "powershell.exe -noexit -command `"echo 'y' | .\opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT\plugins\${{ env.PLUGIN_NAME }}\tools\install_demo_configuration.bat`""
+          Get-Content .\setup.bat
 
       - name: Run Opensearch with A Single Plugin
         uses: ./.github/actions/start-opensearch-with-one-plugin
         with:
-          opensearch-version: 3.0.0
-          plugin-name: opensearch-security
-          plugin-start-script: install_demo_configuration
-          docker-host-plugin-zip: security-plugin.zip
+          opensearch-version: ${{ env.OPENSEARCH_VERSION }}
+          plugin-name: ${{ env.PLUGIN_NAME }}
+          setup-script-name: setup
 
       - name: Run sanity tests
         uses: gradle/gradle-build-action@v2

README.md

@@ -114,6 +114,22 @@ It is common practice to create new transport actions to perform different tasks
 2. Register the action in the [OpenSearch Security plugin](https://github.com/opensearch-project/security). Each new action is registered in the plugin as a new permission. Usually, plugins will define different roles for their plugin (e.g., read-only access, write access). Each role will contain a set of permissions. An example of adding a new permission to the `anomaly_read_access` role for the [Anomaly Detection plugin](https://github.com/opensearch-project/anomaly-detection) can be found in [this PR](https://github.com/opensearch-project/security/pull/997/files).
 3. Register the action in the [OpenSearch Dashboards Security plugin](https://github.com/opensearch-project/security-dashboards-plugin). This plugin maintains the full list of possible permissions, so users can see all of them when creating new roles or searching permissions via Dashboards. An example of adding different permissions can be found in [this PR](https://github.com/opensearch-project/security-dashboards-plugin/pull/689/files).
 
+
+### System Index Protection
+
+The Security Plugin provides protection to system indices used by plugins. The system index names must be explicitly registered in `opensearch.yml` under the `plugins.security.system_indices.indices` setting. See below for an example setup of system index protection from the demo configuration:
+
+```
+plugins.security.system_indices.enabled: true
+plugins.security.system_indices.indices: [".plugins-ml-model", ".plugins-ml-task", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".opendistro-asynchronous-search-response*", ".replication-metadata-store"]
+```
+
+The demo configuration can be modified in the following files to add a new system index to the demo configuration:
+
+- https://github.com/opensearch-project/security/blob/main/tools/install_demo_configuration.sh
+- https://github.com/opensearch-project/security/blob/main/tools/install_demo_configuration.bat
+
+
 ## Contributing
 
 See [developer guide](DEVELOPER_GUIDE.md) and [how to contribute to this project](CONTRIBUTING.md).

build.gradle

@@ -262,6 +262,7 @@ configurations {
             force "io.netty:netty-handler:${versions.netty}"
             force "io.netty:netty-transport:${versions.netty}"
             force "io.netty:netty-transport-native-unix-common:${versions.netty}"
+            force "org.apache.bcel:bcel:6.6.0" // This line should be removed once Spotbugs is upgraded to 4.7.4
         }
     }
 
@@ -450,6 +451,7 @@ dependencies {
     integrationTestImplementation('org.awaitility:awaitility:4.2.0') {
         exclude(group: 'org.hamcrest', module: 'hamcrest')
     }
+    integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.9'
 }
 
 jar {

src/integrationTest/java/org/opensearch/security/SearchOperationTest.java

@@ -121,6 +121,7 @@
 import static org.hamcrest.Matchers.nullValue;
 import static org.opensearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions.Type.ADD;
 import static org.opensearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions.Type.REMOVE;
+import static org.opensearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions.Type.REMOVE_INDEX;
 import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.opensearch.client.RequestOptions.DEFAULT;
 import static org.opensearch.rest.RestRequest.Method.DELETE;
@@ -156,6 +157,7 @@
 import static org.opensearch.test.framework.matcher.BulkResponseMatchers.bulkResponseContainExceptions;
 import static org.opensearch.test.framework.matcher.BulkResponseMatchers.failureBulkResponse;
 import static org.opensearch.test.framework.matcher.BulkResponseMatchers.successBulkResponse;
+import static org.opensearch.test.framework.matcher.ClusterMatchers.aliasExists;
 import static org.opensearch.test.framework.matcher.ClusterMatchers.clusterContainSuccessSnapshot;
 import static org.opensearch.test.framework.matcher.ClusterMatchers.clusterContainTemplate;
 import static org.opensearch.test.framework.matcher.ClusterMatchers.clusterContainTemplateWithAlias;
@@ -211,6 +213,8 @@ public class SearchOperationTest {
 	public static final String INDEX_NAME_SONG_TRANSCRIPTION_JAZZ = "song-transcription-jazz";
 
 	public static final String MUSICAL_INDEX_TEMPLATE = "musical-index-template";
+	public static final String ALIAS_CREATE_INDEX_WITH_ALIAS_POSITIVE = "alias_create_index_with_alias_positive";
+	public static final String ALIAS_CREATE_INDEX_WITH_ALIAS_NEGATIVE = "alias_create_index_with_alias_negative";
 
 	public static final String UNDELETABLE_TEMPLATE_NAME = "undeletable-template-name";
 
@@ -299,16 +303,19 @@ public class SearchOperationTest {
 							"indices:admin/create", "indices:admin/get", "indices:admin/delete", "indices:admin/close",
 							"indices:admin/close*", "indices:admin/open", "indices:admin/resize", "indices:monitor/stats",
 							"indices:monitor/settings/get", "indices:admin/settings/update", "indices:admin/mapping/put",
-							"indices:admin/mappings/get", "indices:admin/cache/clear"
+							"indices:admin/mappings/get", "indices:admin/cache/clear", "indices:admin/aliases"
 					)
 					.on(INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("*")));
 
+	private static User USER_ALLOWED_TO_CREATE_INDEX = new User("user-allowed-to-create-index")
+		.roles(new Role("create-index-role").indexPermissions("indices:admin/create").on("*"));
+
 	@ClassRule
 	public static final LocalCluster cluster = new LocalCluster.Builder()
 		.clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS).anonymousAuth(false)
 		.authc(AUTHC_HTTPBASIC_INTERNAL)
 		.users(ADMIN_USER, LIMITED_READ_USER, LIMITED_WRITE_USER, DOUBLE_READER_USER, REINDEXING_USER, UPDATE_DELETE_USER,
-			USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES)
+			USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES, USER_ALLOWED_TO_CREATE_INDEX)
 		.audit(new AuditConfiguration(true)
 			.compliance(new AuditCompliance().enabled(true))
 			.filters(new AuditFilters().enabledRest(true).enabledTransport(true))
@@ -362,7 +369,9 @@ public void cleanData() throws ExecutionException, InterruptedException {
 			}
 		}
 
-		for(String aliasToBeDeleted : List.of(TEMPORARY_ALIAS_NAME, ALIAS_USED_IN_MUSICAL_INDEX_TEMPLATE_0001, ALIAS_USED_IN_MUSICAL_INDEX_TEMPLATE_0002)) {
+		List<String> aliasesToBeDeleted = List.of(TEMPORARY_ALIAS_NAME, ALIAS_USED_IN_MUSICAL_INDEX_TEMPLATE_0001,
+			ALIAS_USED_IN_MUSICAL_INDEX_TEMPLATE_0002, ALIAS_CREATE_INDEX_WITH_ALIAS_POSITIVE, ALIAS_CREATE_INDEX_WITH_ALIAS_NEGATIVE);
+		for(String aliasToBeDeleted : aliasesToBeDeleted) {
 			if(indices.exists(new IndicesExistsRequest(aliasToBeDeleted)).get().isExists()) {
 				AliasActions aliasAction = new AliasActions(AliasActions.Type.REMOVE).indices(SONG_INDEX_NAME).alias(aliasToBeDeleted);
 				internalClient.admin().indices().aliases(new IndicesAliasesRequest().addAliasAction(aliasAction)).get();
@@ -1838,6 +1847,34 @@ public void deleteIndex_negative() throws IOException {
 		}
 	}
 
+	@Test
+	//required permissions: indices:admin/aliases, indices:admin/delete
+	public void shouldDeleteIndexByAliasRequest_positive() throws IOException {
+		String indexName = INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("delete_index_by_alias_request_positive");
+		IndexOperationsHelper.createIndex(cluster, indexName);
+		try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES)) {
+			IndicesAliasesRequest request = new IndicesAliasesRequest().addAliasAction(new AliasActions(REMOVE_INDEX).indices(indexName));
+
+			var response = restHighLevelClient.indices().updateAliases(request, DEFAULT);
+
+			assertThat(response.isAcknowledged(), is(true));
+			assertThat(cluster, not(indexExists(indexName)));
+		}
+		auditLogsRule.assertExactlyOne(userAuthenticated(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES).withRestRequest(POST, "/_aliases"));
+		auditLogsRule.assertExactly(2, grantedPrivilege(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES, "IndicesAliasesRequest"));
+		auditLogsRule.assertExactly(2, auditPredicate(INDEX_EVENT).withEffectiveUser(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES));
+	}
+
+	@Test
+	public void shouldDeleteIndexByAliasRequest_negative() throws IOException {
+		String indexName = "delete_index_by_alias_request_negative";
+		try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES)) {
+			IndicesAliasesRequest request = new IndicesAliasesRequest().addAliasAction(new AliasActions(REMOVE_INDEX).indices(indexName));
+
+			assertThatThrownBy(() -> restHighLevelClient.indices().updateAliases(request, DEFAULT), statusException(FORBIDDEN));
+		}
+	}
+
 	@Test
 	//required permissions: "indices:admin/get"
 	public void getIndex_positive() throws IOException {
@@ -2274,4 +2311,38 @@ public void clearIndexCache_negative() throws IOException {
 			);
 		}
 	}
+
+	@Test
+	//required permissions: "indices:admin/create", "indices:admin/aliases"
+	public void shouldCreateIndexWithAlias_positive() throws IOException {
+		String indexName = INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("create_index_with_alias_positive");
+		try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES)) {
+			CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName)
+				.alias(new Alias(ALIAS_CREATE_INDEX_WITH_ALIAS_POSITIVE));
+
+			CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(createIndexRequest, DEFAULT);
+
+			assertThat(createIndexResponse, isSuccessfulCreateIndexResponse(indexName));
+			assertThat(cluster, indexExists(indexName));
+			assertThat(internalClient, aliasExists(ALIAS_CREATE_INDEX_WITH_ALIAS_POSITIVE));
+		}
+		auditLogsRule.assertExactlyOne(userAuthenticated(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES).withRestRequest(PUT, "/index_operations_create_index_with_alias_positive"));
+		auditLogsRule.assertExactly(2, grantedPrivilege(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES, "CreateIndexRequest"));
+		auditLogsRule.assertExactly(2, auditPredicate(INDEX_EVENT).withEffectiveUser(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES));
+	}
+
+	@Test
+	public void shouldCreateIndexWithAlias_negative() throws IOException {
+		String indexName = INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("create_index_with_alias_negative");
+		try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(USER_ALLOWED_TO_CREATE_INDEX)) {
+			CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName)
+				.alias(new Alias(ALIAS_CREATE_INDEX_WITH_ALIAS_NEGATIVE));
+
+			assertThatThrownBy(() -> restHighLevelClient.indices().create(createIndexRequest, DEFAULT), statusException(FORBIDDEN));
+
+			assertThat(internalClient, not(aliasExists(ALIAS_CREATE_INDEX_WITH_ALIAS_NEGATIVE)));
+		}
+		auditLogsRule.assertExactlyOne(userAuthenticated(USER_ALLOWED_TO_CREATE_INDEX).withRestRequest(PUT, "/index_operations_create_index_with_alias_negative"));
+		auditLogsRule.assertExactlyOne(missingPrivilege(USER_ALLOWED_TO_CREATE_INDEX, "CreateIndexRequest"));
+	}
 }