Skip to content
generate and commit wordlists

generate and commit wordlists #72

Sign in to view logs

generate and commit wordlists

generate and commit wordlists #72

Workflow file for this run

.github/workflows/wordlists.yml at 74714ce
name: generate and commit wordlists
# on: workflow_dispatch
on:
schedule:
- cron: "00 12 28 * *" # At 12:00 on day-of-month 28.
jobs:
wordlists:
name: generate new wordlists
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v2
- name: Set current date as env variable
run: echo "NOW=$(date +'%Y_%m_%d')" >> $GITHUB_ENV
- name: put service account creds in a file
run: echo '${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}' > ./credentials.json
- name: make wordlists dir
run: mkdir -p data/automated/
- name: make technologies dir
run: mkdir -p data/technologies/
- name: wget
run: wget -r --no-parent -R "index.html*" https://wordlists-cdn.assetnote.io/ -nH
- name: generate ./data/automated/httparchive_parameters_top_1m_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} parameters --output "./data/automated/httparchive_parameters_top_1m_${{ env.NOW }}.txt" --limit 1000000 --sources httparchive
- name: sanity check
run: head -n25 "./data/automated/httparchive_parameters_top_1m_${{ env.NOW }}.txt"
- name: generate ./data/automated/httparchive_apiroutes_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} apiroutes --output "./data/automated/httparchive_apiroutes_${{ env.NOW }}.txt" --limit 1000000 --sources httparchive
- name: sanity check
run: head -n25 "./data/automated/httparchive_apiroutes_${{ env.NOW }}.txt"
- name: generate ./data/automated/httparchive_php_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} ext-wordlist --output "./data/automated/httparchive_php_${{ env.NOW }}.txt" --limit 10000000 --sources httparchive -e php
- name: sanity check
run: head -n25 "./data/automated/httparchive_php_${{ env.NOW }}.txt"
- name: generate ./data/automated/httparchive_jsp_jspa_do_action_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} ext-wordlist --output "./data/automated/httparchive_jsp_jspa_do_action_${{ env.NOW }}.txt" --limit 10000000 --sources httparchive -e jsp,jspa,do,action
- name: sanity check
run: head -n25 "./data/automated/httparchive_jsp_jspa_do_action_${{ env.NOW }}.txt"
- name: generate ./data/automated/httparchive_aspx_asp_cfm_svc_ashx_asmx_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} ext-wordlist --output "./data/automated/httparchive_aspx_asp_cfm_svc_ashx_asmx_${{ env.NOW }}.txt" --limit 10000000 --sources httparchive -e aspx,asp,cfm,svc,ashx,asmx
- name: sanity check
run: head -n25 "./data/automated/httparchive_aspx_asp_cfm_svc_ashx_asmx_${{ env.NOW }}.txt"
- name: generate ./data/automated/httparchive_html_htm_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} ext-wordlist --output "./data/automated/httparchive_html_htm_${{ env.NOW }}.txt" --limit 10000000 --sources httparchive -e html,htm
- name: sanity check
run: head -n25 "./data/automated/httparchive_html_htm_${{ env.NOW }}.txt"
- name: generate ./data/automated/httparchive_js_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} ext-wordlist --output "./data/automated/httparchive_js_${{ env.NOW }}.txt" --limit 10000000 --sources httparchive -e js
- name: sanity check
run: head -n25 "./data/automated/httparchive_js_${{ env.NOW }}.txt"
- name: generate ./data/automated/httparchive_cgi_pl_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} ext-wordlist --output "./data/automated/httparchive_cgi_pl_${{ env.NOW }}.txt" --limit 10000000 --sources httparchive -e cgi,pl
- name: sanity check
run: head -n25 "./data/automated/httparchive_cgi_pl_${{ env.NOW }}.txt"
- name: generate ./data/automated/httparchive_xml_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} ext-wordlist --output "./data/automated/httparchive_xml_${{ env.NOW }}.txt" --limit 10000000 --sources httparchive -e xml
- name: sanity check
run: head -n25 "./data/automated/httparchive_xml_${{ env.NOW }}.txt"
- name: generate ./data/automated/httparchive_txt_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} ext-wordlist --output "./data/automated/httparchive_txt_${{ env.NOW }}.txt" --limit 10000000 --sources httparchive -e txt
- name: sanity check
run: head -n25 "./data/automated/httparchive_txt_${{ env.NOW }}.txt"
- name: generate ./data/automated/httparchive_directories_1m_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} directories --output "./data/automated/httparchive_directories_1m_${{ env.NOW }}.txt" --limit 1000000 --sources httparchive
- name: sanity check
run: head -n25 "./data/automated/httparchive_directories_1m_${{ env.NOW }}.txt"
- name: generate ./data/automated/httparchive_subdomains_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} subdomains --output "./data/automated/httparchive_subdomains_${{ env.NOW }}.txt" --limit 10000000 --sources httparchive
- name: sanity check
run: head -n25 "./data/automated/httparchive_subdomains_${{ env.NOW }}.txt"
- name: generate ./data/technologies/httparchive_adobe_experience_manager_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} technologies --output "./data/technologies/httparchive_adobe_experience_manager_${{ env.NOW }}.txt" --limit 10000000 --technology "Adobe Experience Manager"
- name: sanity check
run: head -n25 "./data/technologies/httparchive_adobe_experience_manager_${{ env.NOW }}.txt"
- name: generate ./data/technologies/httparchive_spring_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} technologies --output "./data/technologies/httparchive_spring_${{ env.NOW }}.txt" --limit 10000000 --technology "Spring"
- name: sanity check
run: head -n25 "./data/technologies/httparchive_spring_${{ env.NOW }}.txt"
- name: generate ./data/technologies/httparchive_symfony_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} technologies --output "./data/technologies/httparchive_symfony_${{ env.NOW }}.txt" --limit 10000000 --technology "Symfony"
- name: sanity check
run: head -n25 "./data/technologies/httparchive_symfony_${{ env.NOW }}.txt"
- name: generate ./data/technologies/httparchive_coldfusion_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} technologies --output "./data/technologies/httparchive_coldfusion_${{ env.NOW }}.txt" --limit 10000000 --technology "Adobe ColdFusion"
- name: sanity check
run: head -n25 "./data/technologies/httparchive_coldfusion_${{ env.NOW }}.txt"
- name: generate ./data/technologies/httparchive_yii_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} technologies --output "./data/technologies/httparchive_yii_${{ env.NOW }}.txt" --limit 10000000 --technology "Yii"
- name: sanity check
run: head -n25 "./data/technologies/httparchive_yii_${{ env.NOW }}.txt"
- name: generate ./data/technologies/httparchive_laravel_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} technologies --output "./data/technologies/httparchive_laravel_${{ env.NOW }}.txt" --limit 10000000 --technology "Laravel"
- name: sanity check
run: head -n25 "./data/technologies/httparchive_laravel_${{ env.NOW }}.txt"
- name: generate ./data/technologies/httparchive_rails_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} technologies --output "./data/technologies/httparchive_rails_${{ env.NOW }}.txt" --limit 10000000 --technology "Ruby on Rails"
- name: sanity check
run: head -n25 "./data/technologies/httparchive_rails_${{ env.NOW }}.txt"
- name: generate ./data/technologies/httparchive_tomcat_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} technologies --output "./data/technologies/httparchive_tomcat_${{ env.NOW }}.txt" --limit 10000000 --technology "Apache Tomcat"
- name: sanity check
run: head -n25 "./data/technologies/httparchive_tomcat_${{ env.NOW }}.txt"
- name: generate ./data/technologies/httparchive_nginx_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} technologies --output "./data/technologies/httparchive_nginx_${{ env.NOW }}.txt" --limit 10000000 --technology "Nginx"
- name: sanity check
run: head -n25 "./data/technologies/httparchive_nginx_${{ env.NOW }}.txt"
- name: generate ./data/technologies/httparchive_apache_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} technologies --output "./data/technologies/httparchive_apache_${{ env.NOW }}.txt" --limit 10000000 --technology "Apache"
- name: sanity check
run: head -n25 "./data/technologies/httparchive_apache_${{ env.NOW }}.txt"
- name: generate ./data/technologies/httparchive_zend_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} technologies --output "./data/technologies/httparchive_zend_${{ env.NOW }}.txt" --limit 10000000 --technology "Zend"
- name: sanity check
run: head -n25 "./data/technologies/httparchive_zend_${{ env.NOW }}.txt"
- name: generate ./data/technologies/httparchive_cherrypy_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} technologies --output "./data/technologies/httparchive_cherrypy_${{ env.NOW }}.txt" --limit 10000000 --technology "CherryPy"
- name: sanity check
run: head -n25 "./data/technologies/httparchive_cherrypy_${{ env.NOW }}.txt"
- name: generate ./data/technologies/httparchive_flask_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} technologies --output "./data/technologies/httparchive_flask_${{ env.NOW }}.txt" --limit 10000000 --technology "Flask"
- name: sanity check
run: head -n25 "./data/technologies/httparchive_flask_${{ env.NOW }}.txt"
- name: generate ./data/technologies/httparchive_django_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} technologies --output "./data/technologies/httparchive_django_${{ env.NOW }}.txt" --limit 10000000 --technology "Django"
- name: sanity check
run: head -n25 "./data/technologies/httparchive_django_${{ env.NOW }}.txt"
- name: generate ./data/technologies/httparchive_express_${{ env.NOW }}.txt
run: |
sudo ./commonspeak2 --credentials credentials.json --project ${{ secrets.GCP_PROJECT_ID }} technologies --output "./data/technologies/httparchive_express_${{ env.NOW }}.txt" --limit 10000000 --technology "Express"
- name: sanity check
run: head -n25 "./data/technologies/httparchive_express_${{ env.NOW }}.txt"
- name: generate automated json
run: python gen-json.py ./data/automated/ > ./data/automated.json
- name: generate technologies json
run: python gen-json.py ./data/technologies/ > ./data/technologies.json
- name: Copy data folder content recursively to remote
uses: garygrossgarten/github-action-scp@release
with:
local: data
remote: /var/www/html/data/
host: ${{ secrets.SSH_HOST }}
username: ${{ secrets.SSH_USER }}
privateKey: ${{ secrets.SSH_PRIVATE_KEY }}
concurrency: 5
- name: setup ssh
env:
SSH_AUTH_SOCK: /tmp/ssh_agent.sock
run: |
mkdir -p ~/.ssh
ssh-keyscan github.com >> ~/.ssh/known_hosts
ssh-agent -a $SSH_AUTH_SOCK > /dev/null
ssh-add - <<< "${{ secrets.SSH_DEPLOY_KEY }}"
- name: git commit and push
env:
SSH_AUTH_SOCK: /tmp/ssh_agent.sock
run: |
git config user.name "commonspeak2-delivery"
git config user.email ""
git add data/*.json
git commit -m "$(date +%Y-%m-%dT%H:%M:%S%z)"
git push origin master
# references
# [1]: https://www.webfactory.de/blog/use-ssh-key-for-private-repositories-in-github-actions