himynamesdave is running cxe2stix #26
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: R2 Daily Upload | |
| run-name: ${{ github.actor }} is running cxe2stix | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 7 * * *" # 7am everyday | |
| jobs: | |
| upload-daily: | |
| runs-on: ubuntu-latest | |
| env: | |
| DATE: ${{ vars.DATE || 'yesterday' }} | |
| services: | |
| redis: | |
| image: redis | |
| options: >- | |
| --health-cmd "redis-cli ping" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 6379:6379 | |
| steps: | |
| - name: install gh | |
| if: ${{ env.ACT }} | |
| run: | | |
| (type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) \ | |
| && sudo mkdir -p -m 755 /etc/apt/keyrings \ | |
| && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ | |
| && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ | |
| && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ | |
| && sudo apt update \ | |
| && sudo apt install gh -y | |
| - id: setup_rclone_config | |
| run: | | |
| rclone_config=$(echo -e '[r2]' \ | |
| '\ntype = s3' \ | |
| '\nprovider = Cloudflare' \ | |
| '\naccess_key_id = ${{ secrets.CLOUDFLARE_ACCESS_KEY_ID }}' \ | |
| '\nsecret_access_key = ${{ secrets.CLOUDFLARE_ACCESS_KEY_SECRET }}' \ | |
| '\nregion = auto' \ | |
| '\nendpoint = ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com' \ | |
| '\nacl = private') | |
| rclone_config_b64=$(base64 -w0 <<< $rclone_config) | |
| echo "::add-mask::$rclone_config_b64" | |
| echo rclone_cf_config=$rclone_config_b64 >> $GITHUB_ENV | |
| - name: Setup Rclone | |
| uses: AnimMouse/setup-rclone@v1 | |
| with: | |
| rclone_config: ${{ secrets.RCLONE_CONFIG || env.rclone_cf_config }} | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'true' | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.12 | |
| - name: Install requirements | |
| run: | | |
| set -e | |
| # create a venv | |
| python -m venv cxe2stix_helper-venv | |
| source cxe2stix_helper-venv/bin/activate | |
| # install requirements | |
| pip install -r requirements.txt | |
| - name: Run CPE2STIX & CVE2STIX | |
| env: | |
| NVD_API_KEY: ${{ secrets.NVD_API_KEY }} | |
| REDIS_HOST: 127.0.0.1 | |
| run: | | |
| set -e | |
| if [[ -z "$NVD_API_KEY" ]]; then | |
| echo NVD_API_KEY must be set && exit 1 | |
| fi | |
| source cxe2stix_helper-venv/bin/activate | |
| YESTERDAY=$(date -u -d $DATE +"%Y-%m-%d") | |
| python3 cxe2stix_helper.py \ | |
| --run_cpe2stix \ | |
| --run_cve2stix \ | |
| --last_modified_earliest "$YESTERDAY"T00:00:00 \ | |
| --last_modified_latest "$YESTERDAY"T23:59:59 \ | |
| --file_time_range 1d | |
| - name: upload bundle to r2 | |
| run: rclone copy output/bundles/ r2:cxe2stix-helper-github-action-output/ -v |