NEW @W-17014756@ Adding functionality to smoke tests

.github/workflows/run-tests.yml

@@ -46,10 +46,35 @@ jobs:
           path: |
             ./reports
             ./coverage
+
+  # The smoke tests should mimic a production setup as closely as possible. To that end, we'll spin up a tarball and
+  # then upload it as an artifact, so the smoke test jobs can pull it down and install it.
+  build-installable-tarball:
+    runs-on: ubuntu-latest
+    steps:
+      # Obviously, we need to get the code first
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.target-branch }}
+      # Make sure we're on the right version of Node
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 'lts/*' # Always use Node LTS for building the tarball
+      # Install/build dependencies
+      - run: yarn
+      - run: yarn build
+      # Create the tarball
+      - run: npm pack
+      # Upload the tarball as an artifact
+      - uses: actions/upload-artifact@v4
+        with:
+          name: smoke-test-tarball
+          path: ./salesforce-plugin-code-analyzer-*.tgz
   # For now, the smoke tests are no-ops, so the job doesn't have to do anything exciting.
   # When smoke-tests stop being no-ops, we'll need a build-dependencies step to create a tarball artifact, and so forth.
   # We'll be able to use the v4 workflow as a template.
   smoke-tests:
+    needs: build-installable-tarball
     strategy:
       # By default, if any job in a matrix fails, all other jobs are automatically cancelled. This makes the jobs run
       # to completion instead.
@@ -59,8 +84,47 @@ jobs:
         os: [{vm: ubuntu-latest, exe: .sh}, {vm: macos-latest, exe: .sh}, {vm: windows-latest, exe: .cmd}]
     runs-on: ${{ matrix.os.vm }}
     steps:
+      # Check out the code
       - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.target-branch }}
-      - run: smoke-tests/smoke-test${{ matrix.os.exe }} sf
+      # Set up Node and Java
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node.version }}
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+      # Install SF CLI via NPM
+      - run: npm install -g @salesforce/cli
+      # Download and install the Tarball artifact
+      - uses: actions/download-artifact@v4
+        id: download
+        with:
+          name: smoke-test-tarball
+          # Download the tarball to a subdirectory of HOME, to guarantee that it's somewhere the installation command
+          # can see
+          path: ~/downloads/tarball
+      - name: Echo tarball download location
+        run: echo ${{ steps.download.outputs.download-path }}
+      - name: Install Tarball
+        shell: bash
+        run: |
+          # We need to determine the Tarball's name first.
+          TARBALL_NAME=$(ls ~/downloads/tarball | grep salesforce-plugin-code-analyzer-5\\.0\\.0-alpha\\.[0-9]*\\.tgz)
+          # We need to determine the Tarball's location in an installable way.
+          # Get the path to the download folder. Swap out backslashes for forward slashes to ensure Windows compatibility.
+          RAW_TARBALL_PATH=`echo '${{ steps.download.outputs.download-path }}' | tr '\\' '/'`
+          # If the path starts with "C:", that needs to be pulled off.
+          ADJUSTED_TARBALL_PATH=`[[ $RAW_TARBALL_PATH = C* ]] && echo $RAW_TARBALL_PATH | cut -d':' -f 2 || echo $RAW_TARBALL_PATH`
+          # Install the tarball, piping in a `y` to simulate agreeing to install an unsigned package. Use the URI of the file's full path.
+          echo y | sf plugins install "file://${ADJUSTED_TARBALL_PATH}/${TARBALL_NAME}"
+      - name: Run smoke tests
+        run: smoke-tests/smoke-test${{ matrix.os.exe }} sf
+      - uses: actions/upload-artifact@v4
+        if: ${{always()}}
+        with:
+          name: smoke-test-results-${{ runner.os }}-node-${{ matrix.node.artifact }}
+          path: smoke-test-results
 

smoke-tests/smoke-test-generator.js

@@ -1,4 +1,5 @@
 const fs = require('fs');
+const path = require('path');
 
 console.log('Switching to smoke test directory');
 process.chdir(__dirname);
@@ -13,7 +14,7 @@ function generateScriptHeader(isBash) {
 
 	// All scripts should start with the same boilerplate disclaimer.
 	header += `${cmt} Auto-generated on ${new Date(Date.now()).toDateString()}
-${cmt} This script WILL EVENTUALLY run a smoke test of the entire plugin by running a series of commands that collectively
+${cmt} This script performs a smoke test of the entire plugin by running a series of commands that collectively
 ${cmt} hit a vertical slice of all major features. If they all succeed, then we can reasonably assume that the plugin is
 ${cmt} approximately stable.
 ${cmt} DO NOT EDIT THIS SCRIPT DIRECTLY! INSTEAD, MAKE CHANGES IN ./smoke-tests/smoke-test-generator.js AND RERUN THAT
@@ -31,11 +32,87 @@ ${cmt} SCRIPT FROM THE PROJECT ROOT!\n`;
 }
 
 function generateScriptBody(isBash, delim) {
+	const exeName = isBash ? '$EXE_NAME' : '%EXE_NAME%';
+
+
 	const commands = [
-		// Log an explainer that since we're in the Alpha stage, we don't have a fully fleshed out smoke test.
-		`echo "At this point in the alpha, the smoke tests are no-ops and it is fine."`
-	]
+		// Log a header.
+		`echo "====== SETUP FOR SMOKE TESTS ======"`,
+		// Delete the smoke test result directory if it already exists
+		`echo "====== Delete a a pre-existing smoke-test-results directory ======"`,
+		isBash ? 'rm -rf smoke-test-results' : 'if exist smoke-test-results rmdir /s /q smoke-test-results || exit /b 1',
+		// Create the output directory
+		`echo "====== Create smoke-test-results directory ======"`,
+		isBash ? 'mkdir -p smoke-test-results' : 'if not exist smoke-test-results mkdir smoke-test-results || exit /b 1',
+		`\n`,
+		`echo "====== SMOKE TESTS FOR CONFIG COMMAND ======"`,
+		// Show the default configuration for all engines
+		`echo "====== Show default config for all engines ======"`,
+		`${exeName} code-analyzer config`,
+		// Show the default configuration for only one engine
+		`echo "====== Show default config for PMD only ======"`,
+		`${exeName} code-analyzer config -r pmd`,
+		// Write the default config for PMD to a file
+		`echo "====== Write PMD's default config to a file ======"`,
+		`${exeName} code-analyzer config -r pmd -f ${path.join('.', 'smoke-test-results', 'pmd-only-config.yml')}`,
+		// Show the configuration from the config file we just created
+		`echo "====== Show configuration from last step's output file ======"`,
+		`${exeName} code-analyzer config -c ${path.join('.', 'smoke-test-results', 'pmd-only-config.yml')}`,
+		`echo "====== Show configuration from pre-existing config file ======"`,
+		`${exeName} code-analyzer config -c ${path.join('.', 'smoke-tests', 'test-data', 'config-files', 'existing-config.yml')}`,
+		`\n`,
+		`echo "====== SMOKE TESTS FOR RULES COMMAND ======"`,
+		// List all rules
+		`echo "====== List all rules ======"`,
+		`${exeName} code-analyzer rules`,
+		// List only rules from ESLint
+		`echo "====== List ESLint rules only ======"`,
+		`${exeName} code-analyzer rules -r eslint`,
+		// List only rules from RetireJS
+		`echo "====== List RetireJS rules only ======"`,
+		`${exeName} code-analyzer rules -r retire-js`,
+		// List only rules that are relevant to a provided workspace
+		`echo "====== List rules relevant to apex-only workspace ======"`,
+		`${exeName} code-analyzer rules -w ${path.join('.', 'smoke-tests', 'test-data', 'workspace-with-apex-files')}`,
+		// List only rules that match nonsensical selector (i.e., list no rules)
+		`echo "====== List rules matching a nonsensical selector (i.e. list no rules) ======"`,
+		`${exeName} code-analyzer rules -r asdfasdfasdf`,
+		// List one rule using a config with overrides
+		`echo "====== List rule overridden in config file ======"`,
+		`${exeName} code-analyzer rules -r no-unsafe-assignment -c ${path.join('.', 'smoke-tests', 'test-data', 'config-files', 'existing-config.yml')}`,
+		'\n',
+		`echo "====== SMOKE TESTS FOR RUN COMMAND ======"`,
+		// Run all rules against a folder
+		`echo "====== Run all rules against a folder ======"`,
+		`${exeName} code-analyzer run -w ${path.join('.', 'smoke-tests', 'test-data', 'workspace-with-mixed-files')}`,
+		// Run all rules against a file
+		`echo "====== Run all rules against a file ======"`,
+		`${exeName} code-analyzer run -w ${path.join('.', 'smoke-tests', 'test-data', 'workspace-with-mixed-files', 'my-script.ts')}`,
+		// Run all rules against a folder and write the output to some files
+		`echo "====== Run all rules against a folder and write to outfiles ======"`,
+		`${exeName} code-analyzer run -w ${path.join('.', 'smoke-tests', 'test-data', 'workspace-with-apex-files')} -f ${path.join('.', 'smoke-test-results', 'outfile.json')} -f ${path.join('.', 'smoke-test-results', 'outfile.html')}`,
+		// Run a selection of rules against a folder
+		`echo "====== Run a selection of rules against a folder ======"`,
+		`${exeName} code-analyzer run -r regex -w ${path.join('.', 'smoke-tests', 'test-data', 'workspace-with-mixed-files')}`,
+		// Run rules using a config file with overrides
+		`echo "====== Run rules using a config file with overrides ======"`,
+		`${exeName} code-analyzer run -c ${path.join('.', 'smoke-tests', 'test-data', 'config-files', 'existing-config.yml')} -w ${path.join('.', 'smoke-tests', 'test-data', 'workspace-with-mixed-files')}`,
+		'\n',
+		`echo "====== CONCLUSION ======"`,
+		`echo "If you are seeing this message, the smoke tests ran successfully, and all is (approximately) well"`
+	];
+
+	// In a cmd script, you need to prepend plugin commands with "call" in order to make sure that the script continues,
+	// and you need to postfix it with another snippet to make it actually exist when an error is encountered.
+	if (!isBash) {
+		for (let i = 0; i < commands.length; i++) {
+			if (commands[i].startsWith(exeName)) {
+				commands[i] = `call ${commands[i]} || exit /b 1`
+			}
+		}
+	}
 
+	// Combine the commands and return the script body.
 	return commands.join('\n');
 }
 

smoke-tests/smoke-test.cmd

@@ -1,9 +1,58 @@
 @echo off
-REM Auto-generated on Tue Aug 27 2024
-REM This script WILL EVENTUALLY run a smoke test of the entire plugin by running a series of commands that collectively
+REM Auto-generated on Fri Oct 18 2024
+REM This script performs a smoke test of the entire plugin by running a series of commands that collectively
 REM hit a vertical slice of all major features. If they all succeed, then we can reasonably assume that the plugin is
 REM approximately stable.
 REM DO NOT EDIT THIS SCRIPT DIRECTLY! INSTEAD, MAKE CHANGES IN ./smoke-tests/smoke-test-generator.js AND RERUN THAT
 REM SCRIPT FROM THE PROJECT ROOT!
 SET EXE_NAME=%1
-echo "At this point in the alpha, the smoke tests are no-ops and it is fine."
+echo "====== SETUP FOR SMOKE TESTS ======"
+echo "====== Delete a a pre-existing smoke-test-results directory ======"
+if exist smoke-test-results rmdir /s /q smoke-test-results || exit /b 1
+echo "====== Create smoke-test-results directory ======"
+if not exist smoke-test-results mkdir smoke-test-results || exit /b 1
+
+
+echo "====== SMOKE TESTS FOR CONFIG COMMAND ======"
+echo "====== Show default config for all engines ======"
+call %EXE_NAME% code-analyzer config || exit /b 1
+echo "====== Show default config for PMD only ======"
+call %EXE_NAME% code-analyzer config -r pmd || exit /b 1
+echo "====== Write PMD's default config to a file ======"
+call %EXE_NAME% code-analyzer config -r pmd -f smoke-test-results/pmd-only-config.yml || exit /b 1
+echo "====== Show configuration from last step's output file ======"
+call %EXE_NAME% code-analyzer config -c smoke-test-results/pmd-only-config.yml || exit /b 1
+echo "====== Show configuration from pre-existing config file ======"
+call %EXE_NAME% code-analyzer config -c smoke-tests/test-data/config-files/existing-config.yml || exit /b 1
+
+
+echo "====== SMOKE TESTS FOR RULES COMMAND ======"
+echo "====== List all rules ======"
+call %EXE_NAME% code-analyzer rules || exit /b 1
+echo "====== List ESLint rules only ======"
+call %EXE_NAME% code-analyzer rules -r eslint || exit /b 1
+echo "====== List RetireJS rules only ======"
+call %EXE_NAME% code-analyzer rules -r retire-js || exit /b 1
+echo "====== List rules relevant to apex-only workspace ======"
+call %EXE_NAME% code-analyzer rules -w smoke-tests/test-data/workspace-with-apex-files || exit /b 1
+echo "====== List rules matching a nonsensical selector (i.e. list no rules) ======"
+call %EXE_NAME% code-analyzer rules -r asdfasdfasdf || exit /b 1
+echo "====== List rule overridden in config file ======"
+call %EXE_NAME% code-analyzer rules -r no-unsafe-assignment -c smoke-tests/test-data/config-files/existing-config.yml || exit /b 1
+
+
+echo "====== SMOKE TESTS FOR RUN COMMAND ======"
+echo "====== Run all rules against a folder ======"
+call %EXE_NAME% code-analyzer run -w smoke-tests/test-data/workspace-with-mixed-files || exit /b 1
+echo "====== Run all rules against a file ======"
+call %EXE_NAME% code-analyzer run -w smoke-tests/test-data/workspace-with-mixed-files/my-script.ts || exit /b 1
+echo "====== Run all rules against a folder and write to outfiles ======"
+call %EXE_NAME% code-analyzer run -w smoke-tests/test-data/workspace-with-apex-files -f smoke-test-results/outfile.json -f smoke-test-results/outfile.html || exit /b 1
+echo "====== Run a selection of rules against a folder ======"
+call %EXE_NAME% code-analyzer run -r regex -w smoke-tests/test-data/workspace-with-mixed-files || exit /b 1
+echo "====== Run rules using a config file with overrides ======"
+call %EXE_NAME% code-analyzer run -c smoke-tests/test-data/config-files/existing-config.yml -w smoke-tests/test-data/workspace-with-mixed-files || exit /b 1
+
+
+echo "====== CONCLUSION ======"
+echo "If you are seeing this message, the smoke tests ran successfully, and all is (approximately) well"

smoke-tests/smoke-test.sh

@@ -1,10 +1,59 @@
 #!/bin/bash
-# Auto-generated on Tue Aug 27 2024
-# This script WILL EVENTUALLY run a smoke test of the entire plugin by running a series of commands that collectively
+# Auto-generated on Fri Oct 18 2024
+# This script performs a smoke test of the entire plugin by running a series of commands that collectively
 # hit a vertical slice of all major features. If they all succeed, then we can reasonably assume that the plugin is
 # approximately stable.
 # DO NOT EDIT THIS SCRIPT DIRECTLY! INSTEAD, MAKE CHANGES IN ./smoke-tests/smoke-test-generator.js AND RERUN THAT
 # SCRIPT FROM THE PROJECT ROOT!
 set -e
 EXE_NAME=$1
-echo "At this point in the alpha, the smoke tests are no-ops and it is fine."
+echo "====== SETUP FOR SMOKE TESTS ======"
+echo "====== Delete a a pre-existing smoke-test-results directory ======"
+rm -rf smoke-test-results
+echo "====== Create smoke-test-results directory ======"
+mkdir -p smoke-test-results
+
+
+echo "====== SMOKE TESTS FOR CONFIG COMMAND ======"
+echo "====== Show default config for all engines ======"
+$EXE_NAME code-analyzer config
+echo "====== Show default config for PMD only ======"
+$EXE_NAME code-analyzer config -r pmd
+echo "====== Write PMD's default config to a file ======"
+$EXE_NAME code-analyzer config -r pmd -f smoke-test-results/pmd-only-config.yml
+echo "====== Show configuration from last step's output file ======"
+$EXE_NAME code-analyzer config -c smoke-test-results/pmd-only-config.yml
+echo "====== Show configuration from pre-existing config file ======"
+$EXE_NAME code-analyzer config -c smoke-tests/test-data/config-files/existing-config.yml
+
+
+echo "====== SMOKE TESTS FOR RULES COMMAND ======"
+echo "====== List all rules ======"
+$EXE_NAME code-analyzer rules
+echo "====== List ESLint rules only ======"
+$EXE_NAME code-analyzer rules -r eslint
+echo "====== List RetireJS rules only ======"
+$EXE_NAME code-analyzer rules -r retire-js
+echo "====== List rules relevant to apex-only workspace ======"
+$EXE_NAME code-analyzer rules -w smoke-tests/test-data/workspace-with-apex-files
+echo "====== List rules matching a nonsensical selector (i.e. list no rules) ======"
+$EXE_NAME code-analyzer rules -r asdfasdfasdf
+echo "====== List rule overridden in config file ======"
+$EXE_NAME code-analyzer rules -r no-unsafe-assignment -c smoke-tests/test-data/config-files/existing-config.yml
+
+
+echo "====== SMOKE TESTS FOR RUN COMMAND ======"
+echo "====== Run all rules against a folder ======"
+$EXE_NAME code-analyzer run -w smoke-tests/test-data/workspace-with-mixed-files
+echo "====== Run all rules against a file ======"
+$EXE_NAME code-analyzer run -w smoke-tests/test-data/workspace-with-mixed-files/my-script.ts
+echo "====== Run all rules against a folder and write to outfiles ======"
+$EXE_NAME code-analyzer run -w smoke-tests/test-data/workspace-with-apex-files -f smoke-test-results/outfile.json -f smoke-test-results/outfile.html
+echo "====== Run a selection of rules against a folder ======"
+$EXE_NAME code-analyzer run -r regex -w smoke-tests/test-data/workspace-with-mixed-files
+echo "====== Run rules using a config file with overrides ======"
+$EXE_NAME code-analyzer run -c smoke-tests/test-data/config-files/existing-config.yml -w smoke-tests/test-data/workspace-with-mixed-files
+
+
+echo "====== CONCLUSION ======"
+echo "If you are seeing this message, the smoke tests ran successfully, and all is (approximately) well"

smoke-tests/test-data/config-files/existing-config.yml

@@ -0,0 +1,20 @@
+
+engines:
+  regex:
+    custom_rules:
+      NoTodos:
+        regex: "/todo/gi"
+        file_extensions:
+          - ".ts"
+        description: "detects todos"
+        tags: ["recommended", "beep"]
+
+rules:
+  pmd:
+    WhileLoopsMustUseBraces:
+      severity: HIGH
+      tags:
+        - CodeStyle
+        - Recommended
+        - apexLanguage
+        - Beep

smoke-tests/test-data/workspace-with-apex-files/MyClass1.cls

@@ -0,0 +1,7 @@
+public class MyClass1 {
+	public static void doSomeQueries(string[] accNames) {
+		for (string accName : accNames) {
+			Account acc = [SELECT Name FROM Account LIMIT 1];
+		}
+	}
+}

smoke-tests/test-data/workspace-with-apex-files/MyClass2.cls

@@ -0,0 +1,5 @@
+public class MyClass2 {
+	public static void doSomeDebugging() {
+		System.debug(LoggingLevel.ERROR, 'aasdfasdfasdasdf');
+	}
+}

smoke-tests/test-data/workspace-with-apex-files/MyClass3.cls

@@ -0,0 +1,7 @@
+public class MyClass3 {
+	public static void declareUnusedVariables() {
+		Account acc = null;
+		boolean b = false;
+		string s = 'asdfasdfasdf';
+	}
+}

smoke-tests/test-data/workspace-with-mixed-files/MyClass1.cls

@@ -0,0 +1,7 @@
+public class MyClass1 {
+	public static void doSomeQueries(string[] accNames) {
+		for (string accName : accNames) {
+			Account acc = [SELECT Name FROM Account LIMIT 1];
+		}
+	}
+}

smoke-tests/test-data/workspace-with-mixed-files/my-script.ts

@@ -0,0 +1,6 @@
+// TODO This is just something to trip a Regex Rule looking for this kind of statement
+
+console.log('asdfasdfasdsasdf');
+const s: string = 'asdfasdf';
+
+console.log(`s is ${s}`);