name: SearchRepos # Controls when the workflow will run on: schedule: # * is a special character in YAML so you have to quote this string - cron: '26 4/12 * * *' # Allows you to run this workflow manually from the Actions tab workflow_dispatch: jobs: search: name: Search runs-on: ubuntu-latest strategy: max-parallel: 10 fail-fast: false matrix: search_str: # comments indicate a snapshot of how many results there were # for a given search around the time it was added to the list # 106 - vuln poc # 110 - heap overflow # 119 - cnvd-c OR cnvd-2 OR cnnvd-2 # 126 - zeroday # 191 - rce poc # 219 - attack poc # 2652 - metasploit module OR payload # 279 - command injection # 383 - 0day # 441 - vulnerability poc # 526 - remote code execution # 960 - sploit # 1114 - cve poc # 2021 - shellcode # 2267 - rce # 4626 - cve-2 OR cve_2 # 18176 - exploit # 17 - UEFI bootkit # 313 - uac bypass # 12 - secure boot bypass # 35 - exploitation persistence # # 2944 # - '"privilege escalation" OR privesc' # # 1504 # - buffer overflow # # 4383 (noisy) # - zero-day # # 21863 (very noisy) # - 0-day # # 502 (noisy) # - 1day steps: - name: Get today and yesterday dates id: date run: | echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT echo "yesterday=$(date -d yesterday '+%Y-%m-%d')" >> $GITHUB_OUTPUT - name: checkout uses: actions/checkout@v4 - name: setup python uses: actions/setup-python@v5 with: python-version: '3.9' - name: setup env uses: ./.github/actions/setup_my_env - name: search github uses: ./.github/actions/single_search with: start_date: ${{ steps.date.outputs.yesterday }} end_date: ${{ steps.date.outputs.today }} search_str: ${{ matrix.search_str }} token: ${{ secrets.GITHUB_TOKEN }} - name: add and commit uses: ./.github/actions/commit_results with: commit_msg: Search for ${{ steps.date.outputs.yesterday }}..${{ steps.date.outputs.today }} ${{ matrix.search_str }} - name: push results uses: ./.github/actions/pull_rebase_push_retry update_summaries: name: Update Summaries needs: search runs-on: ubuntu-latest steps: - name: Get dates id: date run: | echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT echo "yesterday=$(date -d yesterday '+%Y-%m-%d')" >> $GITHUB_OUTPUT echo "this_month=$(date +'%Y-%m')" >> $GITHUB_OUTPUT echo "this_year=$(date +'%Y')" >> $GITHUB_OUTPUT - name: checkout uses: actions/checkout@v4 with: token: ${{ secrets.VUL_DRILLER_TOKEN }} - name: setup python uses: actions/setup-python@v5 with: python-version: '3.9' - name: setup env uses: ./.github/actions/setup_my_env - name: git pull run: git pull - name: summarize yesterday uses: ./.github/actions/generate_summaries with: ymd_option: '--day' ymd_value: ${{ steps.date.outputs.yesterday }} - name: add and commit uses: ./.github/actions/commit_results with: commit_msg: Update Summaries for day ${{ steps.date.outputs.yesterday }} - name: summarize today uses: ./.github/actions/generate_summaries with: ymd_option: '--day' ymd_value: ${{ steps.date.outputs.today }} - name: add and commit uses: ./.github/actions/commit_results with: commit_msg: Update Summaries for day ${{ steps.date.outputs.today }} - name: summarize this month uses: ./.github/actions/generate_summaries with: ymd_option: '--month' ymd_value: ${{ steps.date.outputs.this_month }} - name: add and commit uses: ./.github/actions/commit_results with: commit_msg: Update Summaries for month ${{ steps.date.outputs.this_month }} - name: push results uses: ./.github/actions/pull_rebase_push_retry deep_dive: name: Deep Dive needs: update_summaries runs-on: ubuntu-latest strategy: max-parallel: 11 fail-fast: false matrix: mod: [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] steps: - name: Get dates id: date run: | echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT echo "yesterday=$(date -d yesterday '+%Y-%m-%d')" >> $GITHUB_OUTPUT echo "last_day=$(date -d yesterday +'%d')" >> $GITHUB_OUTPUT echo "this_day=$(date +'%d')" >> $GITHUB_OUTPUT echo "this_month=$(date +'%m')" >> $GITHUB_OUTPUT echo "this_year=$(date +'%Y')" >> $GITHUB_OUTPUT - name: checkout uses: actions/checkout@v4 - name: setup python uses: actions/setup-python@v5 with: python-version: '3.9' - name: setup env uses: ./.github/actions/setup_my_env # in case things have changed since we spawned - name: git pull run: git pull - name: do_deep_dive uses: ./.github/actions/deep_dive with: in_dir: results/${{ steps.date.outputs.this_year }}/${{ steps.date.outputs.this_month }}/${{ steps.date.outputs.last_day }} mod: ${{ matrix.mod }} divisor: ${{ 10 }} token: ${{ secrets.VUL_DRILLER_TOKEN }} max_age: 7200 - name: do_deep_dive uses: ./.github/actions/deep_dive with: in_dir: results/${{ steps.date.outputs.this_year }}/${{ steps.date.outputs.this_month }}/${{ steps.date.outputs.this_day }} mod: ${{ matrix.mod }} divisor: ${{ 10 }} token: ${{ secrets.VUL_DRILLER_TOKEN }} max_age: 3600 - name: add and commit uses: ./.github/actions/commit_results with: results_dir: data commit_msg: Deep Dive for ${{ steps.date.outputs.yesterday }}..${{ steps.date.outputs.today }} chunk ${{ matrix.mod }}/10 - name: push result uses: ./.github/actions/pull_rebase_push_retry repo2vulid: name: Repo to Vul IDs needs: deep_dive runs-on: ubuntu-latest steps: - name: checkout uses: actions/checkout@v4 - name: setup python uses: actions/setup-python@v5 with: python-version: '3.9' - name: setup env uses: ./.github/actions/setup_my_env # in case things have changed since we spawned - name: git pull run: git pull - name: extract vul ids run: repo_to_vul_id --verbose --input_dir data/repo_id - name: add and commit uses: ./.github/actions/commit_results with: results_dir: data/vul_id commit_msg: Update vulnerability ID data - name: push result uses: ./.github/actions/pull_rebase_push_retry