more advance (#206)

* more advance

* more advance

.github/dependabot.yml

@@ -1,27 +1,40 @@
 version: 2
 updates:
-- package-ecosystem: gomod
+- package-ecosystem: "gomod"
   directory: "/"
   schedule:
-    interval: daily
+    interval: "monthly"
   open-pull-requests-limit: 10
-- package-ecosystem: gomod
+  groups:
+    gomod-minor-updates:
+      update-types:
+        - "minor"
+        - "patch"
+- package-ecosystem: "gomod"
   directory: "/function/loader"
   schedule:
-    interval: daily
-  open-pull-requests-limit: 10
-- package-ecosystem: gomod
-  directory: "/sandboxes/staticanalysis"
-  schedule:
-    interval: daily
+    interval: "monthly"
   open-pull-requests-limit: 10
+  groups:
+    loader-minor-updates:
+      update-types:
+        - "minor"
+        - "patch"
 - package-ecosystem: "github-actions"
   directory: "/"
   schedule:
-    interval: "daily"
+    interval: "monthly"
   groups:
     actions-minor-updates:
       update-types:
         - "minor"
         - "patch"
-
+- package-ecosystem: "npm"
+  directory: "/internal/staticanalysis/parsing"
+  schedule:
+    interval: "monthly"
+  groups:
+    parsing-minor-updates:
+      update-types:
+        - "minor"
+        - "patch"

.github/workflows/build-images.yml

@@ -28,4 +28,4 @@ jobs:
         docker version -f '{{.Server.Experimental}}'
 
     - name: build_docker
-      run: make build_prod_images
+      run: make build

.github/workflows/build.yml

@@ -16,9 +16,9 @@ jobs:
         go-version: '1.21.0'
     - name: Install libpcap-dev
       run: sudo apt-get install -y libpcap-dev
-    - run: go build -o scheduler cmd/scheduler/main.go
-    - run: go build -o worker cmd/worker/main.go
-    - run: go build -o analyze cmd/analyze/main.go
+    - run: go build -o scheduler ./cmd/scheduler
+    - run: go build -o worker ./cmd/worker
+    - run: go build -o analyze ./cmd/analyze
     - run: go build -o loader load.go
       working-directory: function/loader
     - run: go build -o staticanalyze staticanalyze.go

.github/workflows/codeql-analysis.yml

@@ -36,6 +36,11 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
+    - name: setup-go
+      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+      with:
+        go-version: '1.21.0'
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@cdcdbb579706841c47f7063dda365e292e5cad7a # v2.13.4

.github/workflows/depsreview.yml

@@ -15,4 +15,4 @@ jobs:
       - name: 'Checkout Repository'
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@4901385134134e04cec5fbe5ddfe3b2c5bd5d976 # v4.0.0
+        uses: actions/dependency-review-action@5bbc3ba658137598168acb2ab73b21c432dd411b # v4.2.5

.github/workflows/osv-scanner-pr.yml

@@ -0,0 +1,19 @@
+name: OSV-Scanner PR Scan
+
+# Change "main" to your default branch if you use a different name, i.e. "master"
+on:
+  pull_request:
+    branches: [ main ]
+  merge_group:
+    branches: [ main ]
+
+# Declare default permissions as read only.
+permissions:
+  # Only need to read contents
+  contents: read
+  # Require writing security events to upload SARIF file to security tab
+  security-events: write
+
+jobs:
+  scan-pr:
+    uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@v1.6.2-beta1"

.github/workflows/osv-scanner-scheduled.yml

@@ -0,0 +1,18 @@
+name: OSV-Scanner Scheduled Scan
+
+on:
+  schedule:
+    - cron: '50 1 * * 6' # run at 01:50 UTC every Saturday
+# Change "main" to your default branch if you use a different name, i.e. "master"
+  push:
+    branches: [ main ]
+
+permissions:
+  # Only need to read contents
+  contents: read
+  # Require writing security events to upload SARIF file to security tab
+  security-events: write
+
+jobs:
+  scan-scheduled:
+    uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@v1.6.2-beta1"

.github/workflows/test.yml

@@ -15,13 +15,13 @@ jobs:
     - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
       with:
         go-version: '1.21.0'
-    - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1
+    - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
       with:
         node-version: 18
     - name: Install libpcap-dev
       run: sudo apt-get install -y libpcap-dev
     - name: Run tests
-      run: go test -v ./...
+      run: go test -v -skip "TestDownload/crates.io_rand_valid_version" ./...
   run-linter:
     runs-on: ubuntu-latest
     steps:
@@ -30,4 +30,4 @@ jobs:
       with:
         go-version: '1.21.0'
     - name: golangci-lint
-      uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v3.7.1
+      uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v4.0.0

.gitignore

@@ -21,4 +21,3 @@
 # these should be pre-installed where needed
 node_modules/
 
-venv

CONTRIBUTING.md

@@ -52,6 +52,8 @@ Prefer smaller PRs to make reviewing easier. Larger changes can be split into sm
 
 ### Code style
 
+We generally follow the [Google Go Style Guide](https://google.github.io/styleguide/go/index).
+
 #### Warnings
 
 Some things that are OK:

Makefile

@@ -19,12 +19,19 @@ else
 	BUILD_ARG=--build-arg=SANDBOX_IMAGE_TAG=$(TAG)
 endif
 
+.PHONY: help
+help:  ## Display this help
+	@awk 'BEGIN {FS = ":.*##"; \
+			printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9\/-]+:.*?##/ \
+			{ printf "  \033[36m%-25s\033[0m %s\n", $$1, $$2 } \
+			/^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
 
 #
 # This recipe builds and pushes images for production. Note: RELEASE_TAG must be set
 #
 .PHONY: cloudbuild
-cloudbuild: require_release_tag push_prod_images
+cloudbuild: require_release_tag push ## Build and push images
 
 .PHONY: require_release_tag
 require_release_tag:
@@ -36,78 +43,80 @@ endif
 #
 # These recipes build all the top-level docker images
 
-build_%_image:
+build/image/%:
 	@# if TAG is 'latest', the two -t arguments are equivalent and do the same thing
 	docker build $(BUILD_ARG) -t ${REGISTRY}/$(IMAGE_NAME) -t ${REGISTRY}/$(IMAGE_NAME):$(TAG) -f $(DOCKERFILE) $(DIR)
 
 #
 # These recipes build the sandbox images.
 #
-build_%_sandbox:
+build/sandbox/%:
 	@# if TAG is 'latest', the two -t arguments are equivalent and do the same thing
 	docker build -t ${REGISTRY}/$(IMAGE_NAME) -t ${REGISTRY}/$(IMAGE_NAME):$(TAG) -f $(DOCKERFILE) $(DIR)
 
-build_analysis_image: DIR=$(PREFIX)
-build_analysis_image: DOCKERFILE=$(PREFIX)/cmd/analyze/Dockerfile
-build_analysis_image: IMAGE_NAME=analysis
+build/image/analysis: DIR=$(PREFIX)
+build/image/analysis: DOCKERFILE=$(PREFIX)/cmd/analyze/Dockerfile
+build/image/analysis: IMAGE_NAME=analysis
 
-build_scheduler_image: DIR=$(PREFIX)
-build_scheduler_image: DOCKERFILE=$(PREFIX)/cmd/scheduler/Dockerfile
-build_scheduler_image: IMAGE_NAME=scheduler
+build/image/scheduler: DIR=$(PREFIX)
+build/image/scheduler: DOCKERFILE=$(PREFIX)/cmd/scheduler/Dockerfile
+build/image/scheduler: IMAGE_NAME=scheduler
 
-build_static_analysis_sandbox: DIR=$(PREFIX)
-build_static_analysis_sandbox: DOCKERFILE=$(SANDBOX_DIR)/staticanalysis/Dockerfile
-build_static_analysis_sandbox: IMAGE_NAME=static-analysis
+build/sandbox/static_analysis: DIR=$(PREFIX)
+build/sandbox/static_analysis: DOCKERFILE=$(SANDBOX_DIR)/staticanalysis/Dockerfile
+build/sandbox/static_analysis: IMAGE_NAME=static-analysis
 
-build_dynamic_analysis_sandbox: DIR=$(SANDBOX_DIR)/dynamicanalysis
-build_dynamic_analysis_sandbox: DOCKERFILE=$(SANDBOX_DIR)/dynamicanalysis/Dockerfile
-build_dynamic_analysis_sandbox: IMAGE_NAME=dynamic-analysis
+build/sandbox/dynamic_analysis: DIR=$(SANDBOX_DIR)/dynamicanalysis
+build/sandbox/dynamic_analysis: DOCKERFILE=$(SANDBOX_DIR)/dynamicanalysis/Dockerfile
+build/sandbox/dynamic_analysis: IMAGE_NAME=dynamic-analysis
 
-.PHONY: build_prod_images
-build_prod_images: build_dynamic_analysis_sandbox build_static_analysis_sandbox build_analysis_image build_scheduler_image
+.PHONY: build
+build: build/sandbox/dynamic_analysis build/sandbox/static_analysis build/image/analysis build/image/scheduler ## Build images
 
 #
 # Builds then pushes analysis and sandbox images
 #
 
-push_%:
+push/image/%:
 	docker push --all-tags ${REGISTRY}/$(IMAGE_NAME)
 
-push_analysis_image: IMAGE_NAME=analysis
-push_analysis_image: build_analysis_image
+push/sandbox/%:
+	docker push --all-tags ${REGISTRY}/$(IMAGE_NAME)
 
-push_scheduler_image: IMAGE_NAME=scheduler
-push_scheduler_image: build_scheduler_image
+push/image/analysis: IMAGE_NAME=analysis
+push/image/analysis: build/image/analysis
 
-push_dynamic_analysis_sandbox: IMAGE_NAME=dynamic-analysis
-push_dynamic_analysis_sandbox: build_dynamic_analysis_sandbox
+push/image/scheduler: IMAGE_NAME=scheduler
+push/image/scheduler: build/image/scheduler
 
-push_static_analysis_sandbox: IMAGE_NAME=static-analysis
-push_static_analysis_sandbox: build_static_analysis_sandbox
+push/sandbox/dynamic_analysis: IMAGE_NAME=dynamic-analysis
+push/sandbox/dynamic_analysis: build/sandbox/dynamic_analysis
 
-.PHONY: push_prod_sandboxes
-push_prod_sandboxes: push_dynamic_analysis_sandbox push_static_analysis_sandbox
+push/sandbox/static_analysis: IMAGE_NAME=static-analysis
+push/sandbox/static_analysis: build/sandbox/static_analysis
 
-.PHONY: push_prod_images
-push_prod_images: push_prod_sandboxes push_analysis_image push_scheduler_image
+.PHONY: push/prod_sandboxes
+push/prod_sandboxes: push/sandbox/dynamic_analysis push/sandbox/static_analysis
 
+.PHONY: push
+push: push/prod_sandboxes push/image/analysis push/image/scheduler ## Push production images
 
 #
 # These update (sync) locally built sandbox images from Docker to
 # podman. In order to use locally built sandbox images for analysis,
 # pass '-nopull' to scripts/run_analysis.sh
 #
-sync_%_sandbox:
-	sudo buildah pull docker-daemon:${REGISTRY}/${IMAGE_NAME}:$(TAG)
+sync/sandbox/%:
+	docker save ${REGISTRY}/${IMAGE_NAME}:$(TAG) | sudo podman load
 
-sync_dynamic_analysis_sandbox: IMAGE_NAME=dynamic-analysis
-sync_dynamic_analysis_sandbox: build_dynamic_analysis_sandbox
+sync/sandbox/dynamic_analysis: IMAGE_NAME=dynamic-analysis
+sync/sandbox/dynamic_analysis: build/sandbox/dynamic_analysis
 
-sync_static_analysis_sandbox: IMAGE_NAME=static-analysis
-sync_static_analysis_sandbox: build_static_analysis_sandbox
+sync/sandbox/static_analysis: IMAGE_NAME=static-analysis
+sync/sandbox/static_analysis: build/sandbox/static_analysis
 
-.PHONY: sync_prod_sandboxes
-sync_prod_sandboxes: sync_dynamic_analysis_sandbox sync_static_analysis_sandbox
+.PHONY: sync
+sync: sync/sandbox/dynamic_analysis sync/sandbox/static_analysis ## Sync prod sandboxes
 
 
 #
@@ -129,11 +138,16 @@ run:
 
 E2E_TEST_COMPOSE_ARGS := -p pa-e2e-testing -f ./configs/e2e/docker-compose.yml -f ./test/e2e/docker-compose.test.yml
 
+.PHONY: e2e_test_build
+e2e_test_build: build_e2e_test_images
+
 .PHONY: e2e_test_start
 e2e_test_start:
 	docker-compose $(E2E_TEST_COMPOSE_ARGS) up -d
 	@echo
 	@echo "To see analysis results, go to http://localhost:9000/minio/package-analysis"
+	@echo "Username: minio"
+	@echo "Password: minio123"
 	@echo
 	@echo "Remember to run 'make e2e_test_stop' when done!"
 	@sleep 5
@@ -160,6 +174,13 @@ e2e_test_logs_scheduler:
 e2e_test_logs_analysis:
 	docker-compose $(E2E_TEST_COMPOSE_ARGS) logs -f analysis
 
+
+.PHONY: build_e2e_test_images
+build_e2e_test_images: TAG=test
+build_e2e_test_images: sync build/image/analysis build/image/scheduler
+
+
+
 .PHONY: test_go
 test_go:
 	go test -v ./...