diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml new file mode 100644 index 0000000000..4a522877d3 --- /dev/null +++ b/.github/workflows/test_security.yml @@ -0,0 +1,104 @@ +name: Security test workflow for k-NN +on: + push: + branches: + - "*" + - "feature/**" + pull_request: + branches: + - "*" + - "feature/**" + +jobs: + Build-ad: + strategy: + matrix: + java: [ 11,17 ] + os: [ubuntu-latest, macos-latest] + fail-fast: false + + name: Security test workflow for k-NN Plugin + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout k-NN + uses: actions/checkout@v1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: Install dependencies on ubuntu + if: startsWith(matrix.os,'ubuntu') + run: | + sudo apt-get install libopenblas-dev gfortran -y + + - name: Install dependencies on macos + if: startsWith(matrix.os, 'macos') + run: | + brew reinstall gcc + export FC=/usr/local/Cellar/gcc/12.2.0/bin/gfortran + + - name: Assemble k-NN + run: | + ./gradlew assemble + # example of variables: + # plugin = opensearch-knn-2.7.0.0-SNAPSHOT.zip + # version = 2.7.0 + # plugin_version = 2.7.0.0 + # qualifier = `SNAPSHOT` + - name: Pull and Run Docker + run: | + plugin=`basename $(ls build/distributions/*.zip)` + version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3` + plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4` + qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1` + if [ $qualifier!=`SNAPSHOT` ] + then + docker_version=$version-$qualifier + else + docker_version=$version + fi + echo plugin version plugin_version qualifier docker_version + echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)" + + cd .. + if docker pull opensearchstaging/opensearch:$docker_version + then + echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile + # this is required knn plugin cannot be deleted until there are plugin that has dependency on it + echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-neural-search ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-neural-search; fi" >> Dockerfile + # TODO review if this is still requied + echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-performance-analyzer ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-performance-analyzer; fi" >> Dockerfile + # saving pre-built artifacts of native libraries as we can't build it with gradle assemle + echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then cp -r /usr/share/opensearch/plugins/opensearch-knn/lib /usr/share/opensearch/knn-libs; fi" >> Dockerfile + echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-knn; fi" >> Dockerfile + echo "ADD k-NN/build/distributions/$plugin /tmp/" >> Dockerfile + echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile + # moving pre-built artifacts of native libraries back to plugin folder + echo "RUN if [ -d /usr/share/opensearch/knn-libs ]; then mv /usr/share/opensearch/knn-libs /usr/share/opensearch/plugins/opensearch-knn/lib; fi" >> Dockerfile + docker build -t opensearch-knn:test . + echo "imagePresent=true" >> $GITHUB_ENV + else + echo "imagePresent=false" >> $GITHUB_ENV + fi + + - name: Run Docker Image + if: env.imagePresent == 'true' + run: | + cd .. + docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-knn:test + sleep 90 + + - name: Run k-NN Integ Test + if: env.imagePresent == 'true' + run: | + security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l` + if [ $security -gt 0 ] + then + echo "Security plugin is available" + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin + else + echo "Security plugin is NOT available, skipping integration tests" + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index ae53f6382e..0174feec0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Add CHANGELOG ([#800](https://github.com/opensearch-project/k-NN/pull/800)) * Bump byte-buddy version from 1.12.22 to 1.14.2 ([#804](https://github.com/opensearch-project/k-NN/pull/804)) * Bump numpy version from 1.22.x to 1.24.2 ([#811](https://github.com/opensearch-project/k-NN/pull/811)) +* Adding github action for secure integ tests ([#836](https://github.com/opensearch-project/k-NN/pull/836)) ### Documentation ### Maintenance ### Refactoring