This repository has been archived by the owner on Mar 24, 2024. It is now read-only.
Scan new pypi pkgs #20398
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scan new pypi pkgs | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '*/20 * * * *' | |
| jobs: | |
| scan_new_pypi_pkgs: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| cache: 'pip' | |
| - run: pip3 install guarddog semgrep | |
| - run: sudo apt-get install yara -y | |
| - name: clone ttp yara rules | |
| run: git clone https://github.com/h4sh5/mal-library-ttps | |
| - name: get new packages names | |
| run: python3 get_packages.py | tee new.txt | |
| - name: download all new packages | |
| run: | | |
| mkdir packages; | |
| cd packages; | |
| cat ../new.txt | parallel pip download --no-deps || echo ignoring some error $? | |
| for i in *; do unzip -- "$i"; done || echo ignorin error $? | |
| - name: run guarddog scan | |
| run: | | |
| guarddog pypi verify -x empty_information -x release_zero -x single_python_file -x repository_integrity_mismatch -x cmd-overwrite --output-format=json new.txt > report.json || echo guarddog error $? | |
| - name: run yara scan | |
| run: yara mal-library-ttps/yara-rules/sus_pkg.yara packages | tee new_yara_results.txt | |
| - name: suspicious files detection | |
| run: | | |
| bash detect_sus_file_formats.sh | |
| - name: high risk pkgs detection | |
| run: | | |
| python3 raise_high_risk_pkgs.py || echo errors occurred here | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: run secret scan | |
| run: semgrep --json -c p/secrets packages/ | tee secrets.jsonl.txt | |
| - name: Push scanned packages to cache | |
| run: | | |
| ls -d packages/* || exit 0 # if there are no new packages scanned, exit | |
| git config --global user.name 'PyPi Scan Bot by h4sh' | |
| git config --global user.email 'pypi-scan-bot-h4sh@open-source.local' | |
| git config --global branch.autoSetupMerge always | |
| git config --global core.mergeoptions --no-edit | |
| git config --global pull.rebase false | |
| git add *.txt *.csv || echo nothing to add | |
| git commit -m "new scan output $(date)" | |
| git pull | |
| git push | |
| - name: Archive report | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: new-scan-report | |
| path: report.json | |