Weekly resource fetching and community filtering #33
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Weekly resource fetching and community filtering | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| #Every Sunday at 8:00 am | |
| - cron: "0 8 * * 0" | |
| # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. | |
| # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. | |
| concurrency: | |
| group: "tools" | |
| cancel-in-progress: false | |
| jobs: | |
| fetch-servers: | |
| runs-on: ubuntu-20.04 | |
| name: Fetch servers | |
| steps: | |
| - name: Checkout main | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install requirement | |
| run: | | |
| python -m pip install -r requirements.txt | |
| sudo apt-get install jq | |
| - name: Fetch list of all available servers | |
| run: | | |
| python sources/bin/get_public_galaxy_servers.py -o sources/data/available_public_servers.csv | |
| - name: Archive available servers | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: available-servers | |
| path: sources/data/available_public_servers.csv | |
| fetch-tools-stepwise: | |
| runs-on: ubuntu-20.04 | |
| name: Fetch tool stepwise | |
| environment: fetch-tools | |
| needs: fetch-servers | |
| strategy: | |
| #max-parallel: 1 #need to run one after another, since otherwise there is a chance, that mulitple jobs want to push to the results branch at the same time (which fails due to merge) | |
| matrix: | |
| subset: | |
| - repositories01.list | |
| - repositories02.list | |
| - repositories03.list | |
| - repositories04.list | |
| steps: | |
| - name: Checkout main | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install requirement | |
| run: python -m pip install -r requirements.txt | |
| - name: Download available servers | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: available-servers | |
| path: sources/data/ | |
| - name: Fetch all tool stepwise | |
| run: | | |
| bash sources/bin/extract_all_tools.sh "${{ matrix.subset }}" | |
| env: | |
| GITHUB_API_KEY: ${{ secrets.GH_API_TOKEN }} | |
| - name: Archive tool sublists production artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: tools-${{ matrix.subset }} | |
| path: communities/all/resources/${{ matrix.subset }}_tools.* | |
| merge-fetch-filter: | |
| runs-on: ubuntu-20.04 | |
| needs: fetch-tools-stepwise | |
| name: Merge tools, fetch tutorials and filter the resources for communities | |
| steps: | |
| - name: Checkout main | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install requirement | |
| run: | | |
| python -m pip install -r requirements.txt | |
| sudo apt-get install jq | |
| - name: Download stepwise tool lists | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: tools-* | |
| merge-multiple: true | |
| path: communities/all/resources/ | |
| - name: Display structure of downloaded files | |
| run: ls -R communities/all/resources/ | |
| - name: Merge all tools | |
| run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry) | |
| awk 'FNR==1 && NR!=1{next;}{print}' communities/all/resources/repositories*.list_tools.tsv > communities/all/resources/tools.tsv | |
| jq -s 'map(.[])' communities/all/resources/repositories*.list_tools.json > communities/all/resources/tools.json | |
| rm communities/all/resources/repositories*.list_tools.json | |
| rm communities/all/resources/repositories*.list_tools.tsv | |
| - name: Generate wordcloud and interactive table | |
| run: | | |
| bash sources/bin/format_tools.sh | |
| - name: Fetch all workflows | |
| run: | | |
| bash sources/bin/extract_all_workflows.sh | |
| - name: Filter workflows for communities | |
| run: | | |
| bash sources/bin/get_community_workflows.sh | |
| - name: Fetch all tutorials | |
| run: | | |
| bash sources/bin/extract_all_tutorials.sh | |
| env: | |
| PLAUSIBLE_API_KEY: ${{ secrets.PLAUSIBLE_API_TOKEN }} | |
| - name: Filter tutorials for communities | |
| run: | | |
| bash sources/bin/get_community_tutorials.sh | |
| #- name: Update tool to keep and exclude for communities | |
| # run: | | |
| # bash sources/bin/update_tools_to_keep_exclude.sh | |
| - name: Filter tools for communities | |
| run: | | |
| bash sources/bin/get_community_tools.sh | |
| - name: Create Pull Request | |
| uses: peter-evans/create-pull-request@v4 | |
| with: | |
| commit-message: Update resources | |
| title: Automatic resources update | |
| body: Automatic resource update done via GitHub Action once a week | |
| base: main | |
| branch: resource-update | |
| delete-branch: true |