Proxy Scraper and Checker #179
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Proxy Scraper and Checker | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 */6 * * *" # Runs every 6 hours; adjust as needed. | |
| jobs: | |
| proxy-spider: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| - uses: actions/checkout@v4 | |
| - uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| check-latest: true | |
| - run: uv run --extra non-termux --no-dev --group nuitka --frozen --python 3.12 python -m nuitka --standalone --python-flag='-m' --assume-yes-for-downloads --lto=yes proxy_spider | |
| - run: mv config.toml proxy_spider.dist/ | |
| - if: ${{ github.event_name != 'pull_request' }} | |
| uses: actions/attest-build-provenance@v1 | |
| with: | |
| subject-path: proxy_spider.dist | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: artifact-${{ matrix.name }} | |
| path: proxy_spider.dist/ | |
| if-no-files-found: error | |
| - name: Configure Proxy Scraper | |
| run: | | |
| # Modify config.ini here if needed; for example: | |
| echo "[proxy]" > config.ini | |
| echo "url = http://example.com/check" >> config.ini | |
| echo "sort_by_speed = true" >> config.ini | |
| echo "geolocation = true" >> config.ini | |
| echo "anonymous_only = false" >> config.ini | |
| echo "output_folder = .out" >> config.ini | |
| - name: Run Proxy Scraper | |
| run: | | |
| sudo chmod +x start.sh | |
| bash start.sh | |
| - name: Push .out Directory to Proxy List Repository | |
| if: success() | |
| run: | | |
| git config --global user.name "github-actions[bot]" | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| git clone https://github.com/threatcode/proxy-list.git proxy-list-repo | |
| cp -r .out/* proxy-list-repo/ | |
| cd proxy-list-repo | |
| git add . | |
| git commit -m "Update proxy list from workflow" | |
| git push https://$GITHUB_TOKEN@github.com/threatcode/proxy-list.git main | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |