feat: smoke tests

.circleci/config.yml

@@ -7,7 +7,7 @@ defaults: &defaults
   parameters:
     node_version:
       type: string
-      default: ""
+      default: ''
   working_directory: ~/snyk
 
 windows_defaults: &windows_defaults
@@ -62,7 +62,7 @@ commands:
     parameters:
       node_version:
         type: string
-        default: ""
+        default: ''
     steps:
       - run:
           name: Install correct version of Node
@@ -203,88 +203,88 @@ workflows:
       - common:
           name: Common
           context: nodejs-install
-          node_version: "8"
+          node_version: '8'
           filters:
             branches:
               ignore:
                 - master
       - danger:
           name: Danger Zone
-          node_version: "8"
+          node_version: '8'
           filters:
             branches:
               ignore:
                 - master
       - test-windows:
           name: Windows Tests for Node v12 support
           context: nodejs-install
-          node_version: "12.0.0"
+          node_version: '12.0.0'
           requires:
-              - Common
-              - Danger Zone
+            - Common
+            - Danger Zone
           filters:
             branches:
               ignore:
                 - master
       - test-windows:
           name: Windows Tests for Node v10 support
           context: nodejs-install
-          node_version: "10.21.0"
+          node_version: '10.21.0'
           requires:
-              - Common
-              - Danger Zone
+            - Common
+            - Danger Zone
           filters:
             branches:
               ignore:
                 - master
       - test-windows:
           name: Windows Tests for Node v8 support
           context: nodejs-install
-          node_version: "8.17.0"
+          node_version: '8.17.0'
           requires:
-              - Common
-              - Danger Zone
+            - Common
+            - Danger Zone
           filters:
             branches:
               ignore:
                 - master
       - test-unix:
           name: Unix Tests for Node v12 support
           context: nodejs-install
-          node_version: "12.16.2"
+          node_version: '12.16.2'
           requires:
-              - Common
-              - Danger Zone
+            - Common
+            - Danger Zone
           filters:
             branches:
               ignore:
                 - master
       - test-unix:
           name: Unix Tests for Node v10 support
           context: nodejs-install
-          node_version: "10"
+          node_version: '10'
           requires:
-              - Common
-              - Danger Zone
+            - Common
+            - Danger Zone
           filters:
             branches:
               ignore:
                 - master
       - test-unix:
           name: Unix Tests for Node v8 support
           context: nodejs-install
-          node_version: "8"
+          node_version: '8'
           requires:
-              - Common
-              - Danger Zone
+            - Common
+            - Danger Zone
           filters:
             branches:
               ignore:
                 - master
       - test-regression:
           name: Regression Test
           context: nodejs-install
-          node_version: "8"
+          node_version: '8'
           requires:
             - Unix Tests for Node v8 support
             - Unix Tests for Node v10 support
@@ -299,7 +299,7 @@ workflows:
       - release:
           name: Release
           context: nodejs-app-release
-          node_version: "10"
+          node_version: '10'
           filters:
             branches:
               only:

.github/workflows/smoke-tests.yml

@@ -0,0 +1,109 @@
+name: Smoke Tests
+
+on:
+  push:
+    branches: [feat/smoke-test]
+  schedule:
+    - cron: '0 * * * *'
+
+jobs:
+  smoke_test:
+    # The type of runner that the job will run on
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        snyk_install_method: [binary, npm]
+        node_version: [8, 10, 12]
+        exclude:
+          - snyk_install_method: binary
+            node_version: 8
+          - snyk_install_method: binary
+            node_version: 10
+        include:
+          - snyk_install_method: binary
+            os: ubuntu-latest
+            snyk_cli_dl_file: snyk-linux
+          - snyk_install_method: binary
+            os: macos-latest
+            snyk_cli_dl_file: snyk-macos
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node_version }}
+
+      - name: Install Snyk with npm
+        if: ${{ matrix.snyk_install_method == 'npm' }}
+        run: |
+          echo "node_version: ${{ matrix.node_version }}"
+          node -v
+          echo "install snyk with npm"
+          npm install -g snyk
+
+      - name: npm install for fixture project
+        working-directory: test/fixtures/basic-npm
+        run: |
+          npm install
+
+      - name: Install Snyk with binary - Non-Windows
+        if: ${{ matrix.snyk_install_method == 'binary' && matrix.os != 'windows-latest' }}
+        run: |
+          echo "install snyk with binary"
+          export latest_version=$(curl -Is "https://github.com/snyk/snyk/releases/latest" | grep location | sed s#.*tag/##g | tr -d "\r")
+          echo "latest_version: ${latest_version}"
+          snyk_cli_dl_linux="https://github.com/snyk/snyk/releases/download/${latest_version}/${{ matrix.snyk_cli_dl_file }}"
+          echo "snyk_cli_dl_linux: ${snyk_cli_dl_linux}"
+          curl -Lo ./snyk-cli $snyk_cli_dl_linux
+          echo "snyk_cli_dl_linux: ${snyk_cli_dl_linux}"
+          chmod -R +x ./snyk-cli
+          sudo mv ./snyk-cli /usr/local/bin/snyk
+          snyk --version
+
+      - name: Install Snyk with binary - Windows
+        if: ${{ matrix.snyk_install_method == 'binary' && matrix.os == 'windows-latest' }}
+        shell: powershell
+        run: |
+          echo "install snyk with binary"
+          echo $env:PATH
+          sh ./test/smoke/install-snyk-binary-win.sh
+
+      - name: Install Shellspec - non-windows
+        if: ${{ matrix.os != 'windows-latest' }}
+        run: |
+          curl -fsSL https://git.io/shellspec | sh -s -- -y
+          sudo ln -s ${HOME}/.local/lib/shellspec/shellspec /usr/local/bin/shellspec
+          ls -la ${HOME}/.local/lib/shellspec
+          echo "shellspec symlink:"
+          ls -la /usr/local/bin/shellspec
+          /usr/local/bin/shellspec --version
+          which shellspec
+          shellspec --version
+
+      - name: Install Shellspec - Windows
+        shell: powershell
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: |
+          Get-Host | Select-Object Version
+          Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux
+          sh ./test/smoke/install-shellspec-win.sh
+
+      - name: Run shellspec tests - non-Windows
+        if: ${{ matrix.os != 'windows-latest' }}
+        working-directory: test/smoke
+        env:
+          SMOKE_TESTS_SNYK_TOKEN: ${{ secrets.SMOKE_TESTS_SNYK_TOKEN }}
+        run: |
+          export EXPECTED_SNYK_VERSION=$(snyk --version)
+          shellspec -f d
+
+      - name: Run shellspec tests - Windows
+        if: ${{ matrix.os == 'windows-latest' }}
+        working-directory: test/smoke
+        shell: powershell
+        env:
+          SMOKE_TESTS_SNYK_TOKEN: ${{ secrets.SMOKE_TESTS_SNYK_TOKEN }}
+        run: |
+          sh ./run-shellspec-win.sh

test/smoke/.shellspec

@@ -0,0 +1 @@
+--require spec_helper

test/smoke/install-shellspec-win.sh

@@ -0,0 +1,5 @@
+echo "install-shellspec-win.sh"
+
+curl -fsSL https://git.io/shellspec | sh -s -- -y
+
+/c/Users/runneradmin/.local/bin/shellspec --version

test/smoke/install-snyk-binary-win.sh

@@ -0,0 +1,11 @@
+echo "install-snyk-binary-win.sh"
+
+export latest_version=$(curl -Is "https://github.com/snyk/snyk/releases/latest" | grep location | sed s#.*tag/##g | tr -d "\r")
+echo "latest_version: ${latest_version}"
+snyk_cli_dl="https://github.com/snyk/snyk/releases/download/${latest_version}/snyk-win.exe"
+echo "snyk_cli_dl: ${snyk_cli_dl}"
+curl -Lo ./snyk-cli.exe $snyk_cli_dl
+./snyk-cli.exe --version
+chmod -R +x ./snyk-cli
+mv ./snyk-cli.exe "/bin/snyk.exe"
+snyk --version

test/smoke/run-shellspec-win.sh

@@ -0,0 +1,5 @@
+echo "run-shellscript-win.sh"
+
+export EXPECTED_SNYK_VERSION=$(snyk --version)
+
+/c/Users/runneradmin/.local/bin/shellspec -f d

test/smoke/spec/snyk_basic_spec.sh

@@ -0,0 +1,73 @@
+#shellcheck shell=sh
+
+Describe "Snyk CLI basics"
+  Describe "snyk version"
+    It "prints version"
+      When run "snyk" version
+      The output should include "${EXPECTED_SNYK_VERSION}"
+      The status should be success
+      # TODO: unusable with our current docker issues
+      The stderr should equal ""
+    End
+
+    It "prints version"
+      When run "snyk" --version
+      The output should include "${EXPECTED_SNYK_VERSION}"
+      The status should be success
+      # TODO: unusable with our current docker issues
+      The stderr should equal ""
+    End
+  End
+
+  Describe "snyk help"
+    It "prints help info"
+      When run snyk help
+      The output should include "$ snyk [command] [options] [package]"
+      The status should be success
+      # TODO: unusable with our current docker issues
+      The stderr should equal ""
+    End
+  End
+
+  Describe "snyk config"
+    It "prints config"
+      When run snyk config
+      The stdout should equal ""
+      The status should be success
+    End
+
+    It "sets config"
+      When run snyk config set newkey=newvalue
+      The output should include "newkey updated"
+      The status should be success
+      The result of "print_snyk_config()" should include "newkey: newvalue"
+    End
+
+    It "unsets config"
+      When run snyk config unset newkey
+      The output should include "newkey deleted"
+      The status should be success
+      The result of "print_snyk_config()" should not include "newkey"
+      The result of "print_snyk_config()" should not include "newvalue"
+    End
+  End
+
+  Describe "snyk auth"
+    It "fails if given bogus token"
+      When run snyk auth abc123
+      The output should include "Authentication failed. Please check the API token"
+      The status should be failure
+      # TODO: unusable with our current docker issues
+      The stderr should equal ""
+    End
+
+    It "updates config file if given legit token"
+      When run snyk auth "${SMOKE_TESTS_SNYK_TOKEN}"
+      The output should include "Your account has been authenticated. Snyk is now ready to be used."
+      The status should be success
+      # TODO: unusable with our current docker issues
+      The stderr should equal ""
+      The result of "print_snyk_config()" should include "api: ${SMOKE_TESTS_SNYK_TOKEN}"
+    End
+  End
+End

test/smoke/spec/snyk_test_spec.sh

@@ -0,0 +1,22 @@
+#shellcheck shell=sh
+
+Describe "Snyk test command"
+  Describe "basic npm test"
+    It "finds vulns in a project"
+      When run snyk test "../fixtures/basic-npm"
+      The status should be failure
+      The output should include "https://snyk.io/vuln/npm:minimatch:20160620"
+      The stderr should equal ""
+    End
+  End
+
+  Describe "basic npm test with JSON output"
+    It "finds vulns in a project"
+      When run snyk test "../fixtures/basic-npm" --json
+      The status should be failure
+      The output should include "npm:minimatch:20160620"
+      The output should include '"vulnerabilities": [' # TODO: check valid JSON? With jq?
+      The stderr should equal ""
+    End
+  End
+End

test/smoke/spec/spec_helper.sh

@@ -0,0 +1,10 @@
+#shellcheck shell=sh
+# set -eu
+
+print_snyk_config() {
+  snyk config
+}
+
+snyk_login() {
+  snyk auth "${SMOKE_TESTS_SNYK_TOKEN}"
+}