Fixing Docker CI for security enabled tests (#710)

Signed-off-by: Amit Galitzky <amgalitz@amazon.com>
opensearch-project · Nov 7, 2022 · bcfe952 · bcfe952
1 parent 06e5edb
commit bcfe952
Show file tree

Hide file tree

Showing 4 changed files with 199 additions and 127 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
diff --git a/.github/workflows/test_build_multi_platform.yml b/.github/workflows/test_build_multi_platform.yml
@@ -0,0 +1,74 @@
+name: Build and Test Anomaly detection
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  Build-ad-windows:
+    strategy:
+      matrix:
+        java: [ 11, 17 ]
+    name: Build and Test Anomaly Detection Plugin on Windows
+    runs-on: windows-latest
+    steps:
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+      - name: Checkout Anomaly Detection
+        uses: actions/checkout@v2
+
+      - name: Build and Run Tests
+        run: |
+          ./gradlew build
+      - name: Publish to Maven Local
+        run: |
+          ./gradlew publishToMavenLocal
+      - name: Multi Nodes Integration Testing
+        run: |
+          ./gradlew integTest -PnumNodes=3
+      - name: Upload Coverage Report
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./build/reports/jacoco/test/jacocoTestReport.xml
+          flags: plugin
+  Build-ad:
+    strategy:
+      matrix:
+        java: [11,17]
+        os: [ubuntu-latest, macos-latest]
+      fail-fast: false
+
+    name: Build and Test Anomaly detection Plugin
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      - name: Checkout AD
+        uses: actions/checkout@v2
+
+      - name: Assemble anomaly-detection
+        run: |
+          ./gradlew assemble
+      - name: Build and Run Tests
+        run: |
+          ./gradlew build -Dtest.logs=true
+      - name: Publish to Maven Local
+        run: |
+          ./gradlew publishToMavenLocal
+      - name: Multi Nodes Integration Testing
+        run: |
+          ./gradlew integTest -PnumNodes=3
+      - name: Upload Coverage Report
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./build/reports/jacoco/test/jacocoTestReport.xml
+          flags: plugin
diff --git a/.github/workflows/test_bwc.yml b/.github/workflows/test_bwc.yml
@@ -0,0 +1,45 @@
+name: Test Anomaly detection BWC
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  Build-ad:
+    strategy:
+      matrix:
+        java: [11,17]
+      fail-fast: false
+
+    name: Test Anomaly detection BWC
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      # anomaly-detection
+      - name: Checkout AD
+        uses: actions/checkout@v2
+
+      - name: Assemble anomaly-detection
+        run: |
+          plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'`
+          echo plugin_version $plugin_version
+          ./gradlew assemble
+          echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."
+          mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
+          echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."
+          ls ./build/distributions/
+          cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
+          echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."
+          ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
+      - name: Run AD Backwards Compatibility Tests
+        run: |
+          echo "Running backwards compatibility tests ..."
+          ./gradlew bwcTestSuite -Dtests.security.manager=false
diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml
@@ -0,0 +1,80 @@
+name: Security test workflow for Anomaly Detection
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  Build-ad:
+    strategy:
+      matrix:
+        java: [11,17]
+      fail-fast: false
+
+    name: Security test workflow for Anomaly Detection
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      # anomaly-detection
+      - name: Checkout AD
+        uses: actions/checkout@v2
+
+      - name: Build Anomaly Detection
+        run: |
+          ./gradlew assemble
+      # example of variables:
+      # plugin = opensearch-anomaly-detection-2.4.0.0-SNAPSHOT.zip
+      # version = 2.4.0, plugin_version = 2.4.0.0, qualifier = SNAPSHOT
+      - name: Pull and Run Docker
+        run: |
+          plugin=`basename $(ls build/distributions/*.zip)`
+          version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3`
+          plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4`
+          qualifier=`echo $plugin|awk -F- '{print $5}'| cut -d. -f 1-1`
+
+          if $qualifier!=SNAPSHOT
+          then
+            docker_version=$version-$qualifier
+          else
+            docker_version=$version
+          fi
+          echo plugin version plugin_version qualifier docker_version
+          echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)"
+
+          cd ..
+          if docker pull opensearchstaging/opensearch:$docker_version
+          then
+            echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
+            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-anomaly-detection ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-anomaly-detection; fi" >> Dockerfile
+            echo "ADD anomaly-detection/build/distributions/$plugin /tmp/" >> Dockerfile
+            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
+            docker build -t opensearch-ad:test .
+            echo "imagePresent=true" >> $GITHUB_ENV
+          else
+            echo "imagePresent=false" >> $GITHUB_ENV
+          fi
+      - name: Run Docker Image
+        if: env.imagePresent == 'true'
+        run: |
+          cd ..
+          docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-ad:test
+          sleep 90
+      - name: Run AD Test
+        if: env.imagePresent == 'true'
+        run: |
+          security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l`
+          if [ $security -gt 0 ]
+          then
+            echo "Security plugin is available"
+            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin
+          else
+            echo "Security plugin is NOT available, skipping integration tests"
+          fi