🛍️ Build 📦 (toolpack_x86_64_Linux) Binaries 📦🗄️ #157
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 🛍️ Build 📦 (toolpack_x86_64_Linux) Binaries 📦🗄️ | |
| # Build x86_64 Linux Binaries on GH action runners | |
| # Main Build Script: https://github.com/Azathothas/Toolpacks/blob/main/.github/scripts/x86_64_Linux/build_debian.sh | |
| # Process :: init_debian.sh --> add_toolpacks.sh --> upload to https://bin.ajam.dev/x86_64_Linux/ | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| # - cron: "30 22 * * 5" # 10:30 PM UTC Friday --> 04:15 AM NPT Mrng Every Saturday | |
| - cron: "30 18 * * 1" # 06:30 PM UTC (12:15 AM NPT Mrng Every Tuesday) | |
| #Sane env | |
| env: | |
| GITHUB_TOKEN: "${{ secrets.TOOLPACKS }}" | |
| PKGFORGE_LOONIX_TOKEN: "${{ secrets.PKGFORGE_LOONIX_TOKEN }}" | |
| RCLONE_CF_R2_PUB: "${{ secrets.RCLONE_CF_R2_PUB }}" | |
| TSKEY: "${{ secrets.TSKEY }}" | |
| UPX_PACK: "YES" | |
| #Begin | |
| jobs: | |
| #------------------------------------------------------------------------------------# | |
| presetup: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
| #------------------------------------------------------------------------------------# | |
| - name: Install Addons | |
| run: | | |
| #presets | |
| set -x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| - name: Setup Env | |
| run: | | |
| ##presets | |
| set +x ; set +e | |
| #-------------# | |
| ##CoreUtils | |
| sudo apt update -y | |
| sudo apt install bc coreutils curl dos2unix fdupes jq moreutils wget -y | |
| sudo apt-get install apt-transport-https apt-utils ca-certificates coreutils dos2unix gnupg2 jq moreutils p7zip-full rename rsync software-properties-common texinfo tmux util-linux wget -y 2>/dev/null ; sudo apt-get update -y 2>/dev/null | |
| ##tmp | |
| SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
| #GH ENV | |
| echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
| #GH Dirs | |
| mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux" | |
| mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux" | |
| #-------------# | |
| mkdir -p "$HOME/bin" | |
| sudo apt update -y | |
| sudo apt install dos2unix -y | |
| ##Setup rClone | |
| mkdir -p "$HOME/.config/rclone" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
| ##User-Agent | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
| echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
| continue-on-error: true | |
| - name: Set TZ to (Asia/Kathmandu) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| sudo apt-get update -y && sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y tzdata | |
| sudo ln -fs "/usr/share/zoneinfo/Asia/Kathmandu" "/etc/localtime" | |
| sudo dpkg-reconfigure --frontend noninteractive tzdata | |
| sudo apt-get install apt-utils software-properties-common -y | |
| sudo apt-get update -y | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| - name: rClone BackUp Repo ("https://pub.ajam.dev/repos/$GITHUB_REPOSITORY") | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| #copy & chdir to Repo | |
| cp -r "$GITHUB_WORKSPACE/main" "$SYSTMP/REPO" && cd "$SYSTMP/REPO" | |
| #Git pull | |
| git pull origin main --no-edit 2>/dev/null | |
| #Del Bloat | |
| rm -rf "$(pwd)/.git" | |
| # rm -rf "$(pwd)/aarch64_arm64_Linux" | |
| # rm -rf "$(pwd)/arm64_v8a_Android" | |
| # rm -rf "$(pwd)/x86_64_Linux" | |
| # rm -rf "$(pwd)/x64_Windows" | |
| #Upload to Pub | |
| echo -e "[+] Syncing $GITHUB_REPOSITORY to pub.ajam.dev/repos/$GITHUB_REPOSITORY \n" | |
| rclone sync "." "r2:/pub/repos/$GITHUB_REPOSITORY/" --user-agent="$USER_AGENT" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| ##Scripts_x86_64_Linux | |
| #Get JQ Module to convert size : https://users.aalto.fi/~tontti/posts/jq-and-human-readable-bytes/ | |
| curl -qfsSL "https://pub.ajam.dev/utils/devscripts/jq/to_human_bytes.jq" -o "./to_human_bytes.jq" | |
| rclone lsjson --fast-list "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/x86_64_Linux/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://pub.ajam.dev/repos/Azathothas/Toolpacks/.github/scripts/x86_64_Linux/bins/\(.Path)"}' | jq . > "./metadata.json.tmp" | |
| #Clean Pretty Formatted | |
| echo "[" $(cat metadata.json.tmp | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json" | |
| #Update Metadata | |
| rclone copyto --checksum "./metadata.json" "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/x86_64_Linux/bins/metadata.json" | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| build-fetch-binaries: | |
| #runs-on: ubuntu-latest | |
| runs-on: amd64-linux-toolpacker | |
| needs: [presetup] | |
| timeout-minutes: 2400 | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
| - name: Install Addons | |
| run: | | |
| #presets | |
| set -x ; set +e | |
| #-------------# | |
| bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| - name: Setup Env | |
| run: | | |
| ##presets | |
| set +x ; set +e | |
| #-------------# | |
| ##CoreUtils | |
| sudo apt update -y | |
| sudo apt install bc coreutils curl dos2unix fdupes jq moreutils wget -y | |
| sudo apt-get install apt-transport-https apt-utils ca-certificates coreutils dos2unix gnupg2 jq moreutils p7zip-full rename rsync software-properties-common texinfo tmux util-linux wget -y 2>/dev/null ; sudo apt-get update -y 2>/dev/null | |
| ##tmp | |
| SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
| #GH ENV | |
| echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
| #GH Dirs | |
| mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux" | |
| mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows" | |
| mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux" | |
| #-------------# | |
| mkdir -p "$HOME/bin" | |
| sudo apt update -y | |
| sudo apt install dos2unix -y | |
| ##Setup rClone | |
| mkdir -p "$HOME/.config/rclone" | |
| echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
| ##User-Agent | |
| USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
| echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
| continue-on-error: true | |
| - name: Set TZ to (Asia/Kathmandu) | |
| run: | | |
| # Presets | |
| set +x ; set +e | |
| #--------------# | |
| sudo apt-get update -y && sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y tzdata | |
| sudo ln -fs "/usr/share/zoneinfo/Asia/Kathmandu" "/etc/localtime" | |
| sudo dpkg-reconfigure --frontend noninteractive tzdata | |
| sudo apt-get install apt-utils software-properties-common -y | |
| sudo apt-get update -y | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #Main Build Script | |
| - name: BUILD | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| ##Main | |
| curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Toolpacks/.github/scripts/x86_64_Linux/build_debian.sh" -o "$SYSTMP/BUILD.sh" | |
| dos2unix --quiet "$SYSTMP/BUILD.sh" ; chmod +xwr "$SYSTMP/BUILD.sh" | |
| ##Run with STDOUT [Slow, Not Recommended] | |
| #bash "$SYSTMP/BUILD.sh" | |
| ##Run with LOGS only | |
| #bash "$SYSTMP/BUILD.sh" > "$SYSTMP/BUILD.log" 2>&1 | |
| ##Run with STDOUT + LOGS | |
| #bash "$SYSTMP/BUILD.sh" | tee "$SYSTMP/BUILD.log" 2>&1 | |
| ##Run without STDOUT/Logs | |
| #bash "$SYSTMP/BUILD.sh" >/dev/null 2>&1 | |
| ##Talscale + Tmux + gotty | |
| curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Github/Runners/Ubuntu/setup_tailscale.sh" -o "$SYSTMP/setup_tailscale.sh" && dos2unix --quiet "$SYSTMP/setup_tailscale.sh" | |
| source "$SYSTMP/setup_tailscale.sh" | |
| tmux new-session -s "toolpacks" -d 'bash "/tmp/BUILD.sh" 2>&1 | tee -a "/tmp/BUILD.log" ; tmux kill-session -t "toolpacks" 2>/dev/null ; exit' | |
| #nohup gotty --address "$TS_IP" --port "7070" --reconnect --reconnect-time 30 tmux attach-session -t "toolpacks" -r >/dev/null 2>&1 & | |
| nohup gotty --address "127.0.0.1" --port "7070" --reconnect --reconnect-time 30 tmux attach-session -t "toolpacks" -r >/dev/null 2>&1 & | |
| sudo tailscale serve --bg="true" --yes="true" --set-path "/tmux" "http://127.0.0.1:7070" | |
| nohup gotty --address "127.0.0.1" --port "7777" --reconnect --reconnect-time 30 btop --tty_on --utf-force >/dev/null 2>&1 & | |
| sudo tailscale serve --bg="true" --yes="true" --set-path "/btop" "http://127.0.0.1:7777" | |
| echo -e "\n\n[+] Tmux TTY: http://$TS_DNS:7070 || https://$TS_DNS/tmux/" | |
| echo -e "[+] BTOP: http://$TS_DNS:7777 || https://$TS_DNS/btop/\n" | |
| ##Loonix (Start) | |
| GH_EVENT="${{ github.event_name }}" | |
| START_TIME="$(date +%s)" && export START_TIME | |
| echo "GH_EVENT=${GH_EVENT}" >> "${GITHUB_ENV}" | |
| echo "START_TIME=${START_TIME}" >> "${GITHUB_ENV}" | |
| rm -rvf "/tmp/PKGFORGE_LOONIX.html" 2>/dev/null | |
| echo -e "ℹ️ <b>Triggered</b> <a href='https://github.com/Azathothas/Toolpacks/blob/main/.github/workflows/build_x86_64_Linux.yaml'><b>🛍️ Build 📦 Bincache (ToolPacks) 📦🗄️</b></a> <a href='https://github.com/Azathothas/Toolpacks'><b>Azathothas/Toolpacks (x86_64-Linux)</b></a> [<b><u>$(date --utc +'%Y-%m-%dT%H:%M:%S.%3N') UTC</u></b>]" > "/tmp/PKGFORGE_LOONIX.html" | |
| echo -e "<b><u>Event</u></b>: <code>${GH_EVENT}</code>" >> "/tmp/PKGFORGE_LOONIX.html" | |
| echo -e "<b><u>Btop</u></b>: https://${TS_DNS}/btop/" >> "/tmp/PKGFORGE_LOONIX.html" | |
| echo -e "<b><u>Tmux</u></b>: https://${TS_DNS}/tmux/" >> "/tmp/PKGFORGE_LOONIX.html" | |
| echo -e "<b><u>GH Repo</u></b>: https://github.com/${GITHUB_REPOSITORY}" >> "/tmp/PKGFORGE_LOONIX.html" | |
| echo -e "<b><u>Maintainer</u></b>: @Azathothas" >> "/tmp/PKGFORGE_LOONIX.html" | |
| curl "https://api.telegram.org/bot${PKGFORGE_LOONIX_TOKEN}/sendMessage" -d "chat_id=-1002007583969&message_thread_id=63949" -d "text=$(cat /tmp/PKGFORGE_LOONIX.html)" -d "parse_mode=html" | |
| set +x ; while tmux has-session -t "toolpacks" ; do sleep 1 ; done | |
| ##Purge Tokens (in case set -x & gh didn't redact) | |
| cat "$SYSTMP/BUILD.log" | ansi2txt > "$SYSTMP/BUILD.log.tmp.txt" | |
| if command -v trufflehog &> /dev/null; then | |
| trufflehog filesystem "$SYSTMP/BUILD.log.tmp.txt" --no-fail --no-verification --no-update --json 2>/dev/null | jq -r '.Raw' | sed '/{/d' | xargs -I "{}" sh -c 'echo "{}" | tr -d " \t\r\f\v"' | xargs -I "{}" sed "s/{}/ /g" -i "$SYSTMP/BUILD.log.tmp.txt" | |
| fi | |
| sed 's/\r/\n/g' -i "$SYSTMP/BUILD.log.tmp.txt" | |
| sed -i '/.*github_pat.*/Id' "$SYSTMP/BUILD.log.tmp.txt" 2>/dev/null | |
| sed -i '/.*ghp_.*/Id' "$SYSTMP/BUILD.log.tmp.txt" 2>/dev/null | |
| sed -i '/.*access_key_id.*/Id' "$SYSTMP/BUILD.log.tmp.txt" 2>/dev/null | |
| sed -i '/.*secret_access_key.*/Id' "$SYSTMP/BUILD.log.tmp.txt" 2>/dev/null | |
| sed -i '/.*token.*/Id' "$SYSTMP/BUILD.log.tmp.txt" 2>/dev/null | |
| sed -i '/.*cloudflarestorage.*/Id' "$SYSTMP/BUILD.log.tmp.txt" 2>/dev/null | |
| grep -viE 'github_pat|ghp_|glpat|hf_|token|access_key_id|secret_access_key|cloudflarestorage' "$SYSTMP/BUILD.log.tmp.txt" > "$SYSTMP/BUILD.log.txt" | |
| ##Fetch Only Relevant Logs | |
| if grep -iq "completed" "$SYSTMP/BUILD.log.txt"; then | |
| mv "$SYSTMP/BUILD.log.txt" "$SYSTMP/BUILD.log" | |
| elif grep -iq "initializing" "$SYSTMP/BUILD.log.txt"; then | |
| mv "$SYSTMP/BUILD.log.txt" "$SYSTMP/BUILD_FAILED.log" | |
| fi | |
| sed -n "$(grep -nE '^\[\+\] (Building|Fetching)' "$SYSTMP/BUILD.log" | head -n 1 | cut -d: -f1),$(grep -nE '^\[\+\] Completed \(Building|Fetching\)' "$SYSTMP/BUILD.log" | tail -n 1 | cut -d: -f1)p" "$SYSTMP/BUILD.log" > "$SYSTMP/BUILD.BIN.log" | |
| ##rClone Upload logs | |
| rclone copyto "$SYSTMP/BUILD.log" "r2:/bin/x86_64_Linux/BUILD.log.txt" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| rclone copyto "$SYSTMP/BUILD_FAILED.log" "r2:/bin/x86_64_Linux/BUILD_FAILED.log.txt" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| rclone copyto "$SYSTMP/BUILD.BIN.log" "r2:/bin/x86_64_Linux/BUILD.BIN.log.txt" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
| continue-on-error: true | |
| - name: Notify Loonix (END) | |
| run: | | |
| ##presets | |
| set +x ; set +e | |
| #-------------# | |
| END_TIME="$(date +%s)" && export END_TIME="${END_TIME}" | |
| ELAPSED_TIME="$(date -u -d@"$((END_TIME - START_TIME))" "+%H(Hr):%M(Min):%S(Sec)")" | |
| echo "ELAPSED_TIME=${ELAPSED_TIME}" >> "${GITHUB_ENV}" | |
| rm -rvf "/tmp/PKGFORGE_LOONIX.html" 2>/dev/null | |
| echo -e "ℹ️ <b>Completed</b> <a href='https://github.com/Azathothas/Toolpacks/blob/main/.github/workflows/build_x86_64_Linux.yaml'><b>🛍️ Build 📦 Bincache (ToolPacks) 📦🗄️</b></a> <a href='https://github.com/Azathothas/Toolpacks'><b>Azathothas/Toolpacks (x86_64-Linux)</b></a> [<b><u>$(date --utc +'%Y-%m-%dT%H:%M:%S.%3N') UTC</u></b>]" > "/tmp/PKGFORGE_LOONIX.html" | |
| echo -e "<b><u>ELAPSED_TIME</u></b>: <code>${ELAPSED_TIME}</code>" >> "/tmp/PKGFORGE_LOONIX.html" | |
| echo -e "<b><u>Event</u></b>: <code>${GH_EVENT}</code>" >> "/tmp/PKGFORGE_LOONIX.html" | |
| echo -e "<b><u>Log (Success)</u></b>: https://bin.pkgforge.dev/x86_64-Linux/BUILD.log.txt" >> "/tmp/PKGFORGE_LOONIX.html" | |
| echo -e "<b><u>Log (Failure)</u></b>: https://bin.pkgforge.dev/x86_64-Linux/BUILD_FAILED.log.txt" >> "/tmp/PKGFORGE_LOONIX.html" | |
| echo -e "<b><u>GH Repo</u></b>: https://github.com/${GITHUB_REPOSITORY}" >> "/tmp/PKGFORGE_LOONIX.html" | |
| echo -e "<b><u>Maintainer</u></b>: @Azathothas" >> "/tmp/PKGFORGE_LOONIX.html" | |
| curl "https://api.telegram.org/bot${PKGFORGE_LOONIX_TOKEN}/sendMessage" -d "chat_id=-1002007583969&message_thread_id=63949" -d "text=$(cat /tmp/PKGFORGE_LOONIX.html)" -d "parse_mode=html" | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #Logs & Artifacts | |
| - name: Upload (x86_64_Linux) Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Toolpack_x86_64_Linux | |
| path: | | |
| /tmp/toolpack_x86_64.7z | |
| /tmp/BUILD.log | |
| compression-level: 0 # no compression, [Default: 6 (GNU Gzip)] | |
| retention-days: 7 | |
| overwrite: true | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# |