Remove scripts/run-test to simplify, call just osemgrep test (#3493)

* Remove scripts/run-test to simplify, call just osemgrep test

It has been almost a month that we run both osemgrep test
and pysemgrep --test and no complaints, so let's remove
the use of pysemgrep --test so we can then remove the
corresponding python code in pysemgrep.

test plan:
make validate
make test-only
wait for green CI checks

* more

.github/workflows/semgrep-rules-test-develop.yml

@@ -1,4 +1,4 @@
-# Running the tests in the repo using `semgrep test --experimental` and
+# Running the tests in the repo using `semgrep test` (osemgrep) and
 # the semgrep/semgrep:pro-develop docker image (the bleeding edge!).
 
 name: semgrep-rules-test-develop
@@ -14,27 +14,20 @@ on:
 jobs:
   test-develop:
     name: rules-test-develop
+    # alt: use directly the semgrep/semgrep:pro-develop container here so we
+    # don't need the calls to 'docker run ...' below
     runs-on: ubuntu-20.04
+    # TODO: remove the with: path: below to simplify
     steps:
     - uses: actions/checkout@v2
       with:
         path: semgrep-rules
+    # alt: call 'make validate' but would require 'make' in the docker image
+    # alt: export SEMGREP="docker run --rm -w ... semgrep"
+    #    make -C "$GITHUB_WORKSPACE"/semgrep-rules validate
+    #TODO: this actually currently fails because of errors in stats/ but GHA
+    # still continue, weird
     - name: run osemgrep validate --pro
       run: docker run --rm -w /src -v ${GITHUB_WORKSPACE}/semgrep-rules:/src semgrep/semgrep:pro-develop semgrep validate --pro .
     - name: run osemgrep test --pro
       run: docker run --rm -w /src -v ${GITHUB_WORKSPACE}/semgrep-rules:/src semgrep/semgrep:pro-develop semgrep test --pro .
-    #TODO: we can delete all the rest below and also scripts/run-tests
-    - name: delete directories not containing rules
-      run: rm -rf semgrep-rules/stats
-    - name: delete rules requiring Semgrep Pro
-      run: rm -rf semgrep-rules/apex semgrep-rules/elixir
-    # TODO: this takes 1m20 in CI and could be optimized by switching to osemgrep
-    - name: validate rules
-      run: |
-        export SEMGREP="docker run --rm -w /src -v ${GITHUB_WORKSPACE}/semgrep-rules:/src semgrep/semgrep:pro-develop semgrep"
-        make -C "$GITHUB_WORKSPACE"/semgrep-rules validate
-    # this now takes 21s with osemgrep instead of 3min with pysemgrep
-    - name: test with semgrep pro develop branch and with --experimental
-      run: |
-        export SEMGREP="docker run --rm -w /src -v ${GITHUB_WORKSPACE}/semgrep-rules:/src semgrep/semgrep:pro-develop semgrep --experimental"
-        make -C "$GITHUB_WORKSPACE"/semgrep-rules test-only

.github/workflows/semgrep-rules-test.yml

@@ -17,13 +17,13 @@ jobs:
     - uses: actions/setup-python@v2
       with:
         python-version: 3.9.2
-    - name: install semgrep
+    - name: install semgrep via pip
       run: pip3 install semgrep
     - name: remove stats directory
       run: rm -rf stats
     - name: remove rules requiring Semgrep Pro
       run: rm -rf apex elixir
     - name: validate rules
-      run: semgrep --validate --config .
-    - name: run semgrep
-      run: semgrep --test --test-ignore-todo
+      run: semgrep validate .
+    - name: run semgrep test
+      run: semgrep test .

Makefile

@@ -1,17 +1,65 @@
 #
 # Check rule validity and check that semgrep finds the expected findings.
+# See https://semgrep.dev/docs/writing-rules/testing-rules for more info.
 #
-# The semgrep repo also runs this as part of its CI for consistency.
+# The semgrep repo (and now semgrep-pro repo) also runs those tests as part
+# of its CI for consistency.
 #
 .PHONY: test
 test:
 	$(MAKE) validate
 	$(MAKE) test-only
 
-.PHONY: validate
-validate:
-	./scripts/run-tests validate
+# Use the SEMGREP env variable to specify a non-standard semgrep command
+SEMGREP ?= semgrep
 
 .PHONY: test-only
+#old: pysemgrep --test was also using flags below but not needed
+# --test-ignore-todo --strict --disable-version-check --metrics=off --verbose
 test-only:
-	./scripts/run-tests test
+	$(SEMGREP) test --pro .
+
+# TODO: semgrep validate use a different targeting than 'semgrep test'
+# so we unfortunately need this whitelist of dirs because it reports
+# errors on stats/ and scripts/ (and .github/workflows/) files otherwise
+# (we also skip libsonnet/ and trusted_python/ which do not contain rules)
+LANG_DIRS=\
+ bash \
+ c \
+ clojure \
+ csharp \
+ dockerfile \
+ generic \
+ go \
+ html \
+ java \
+ javascript \
+ json \
+ kotlin \
+ ocaml \
+ php \
+ python \
+ ruby \
+ rust \
+ scala \
+ solidity \
+ swift \
+ terraform \
+ typescript \
+ yaml
+PRO_DIRS=apex elixir
+OTHER_DIRS=ai problem-based-packs
+DIRS=$(LANG_DIRS) $(PRO_DIRS) $(OTHER_DIRS)
+
+.PHONY: validate
+#old: pysemgrep --validate was also using the flags below but not needed
+# --strict --disable-version-check --metrics=off --verbose
+validate:
+	$(SEMGREP) validate --pro $(DIRS)
+
+.PHONY: test-oss-only
+test-oss-only:
+	@for dir in $(LANG_DIRS) $(OTHER_DIRS); do \
+	   echo "processing $$dir"; \
+           $(SEMGREP) test $$dir; \
+        done