terraform-providers: refactor

Some terraform providers were moved to the `hashicorp` organization,
some are still in the `terraform-providers` organization.

GitHub doesn't show moved repositories when querying via the API (only
on the web interface). In addition, if there are more than 100 results
(there still are), we'd need to implement paging, which is cubersome in curl.

In addition to that, to look up tags, we used to creating a HTTP request
for each and every provider, which caused us to easily get blocked by
GitHubs rate-limiting.

Instead of all that, let's use `hub` (which implements paging) and the
GitHub's GraphQL endpoint to get a list all repositories for a given
organization AND the latest tag for each of them.

We post-process the output with `jq`.

This also removes the providers.txt and folds the data into the script.
The blacklist was becoming too annoying to handle as a config file.

For the whitelist, we still need to individually fetch tags.

Fixes #83215

Co-Authored-By: zimbatm <zimbatm@zimbatm.com>

pkgs/applications/networking/cluster/terraform-providers/update-all

@@ -1,6 +1,7 @@
 #!/usr/bin/env nix-shell
-#! nix-shell -i bash -p bash coreutils curl jq nix
+#! nix-shell -i bash -p bash coreutils jq nix gitAndTools.hub
 # vim: ft=sh sw=2 et
+# shellcheck shell=bash
 #
 # This scripts scans the github terraform-providers repo for new releases,
 # generates the corresponding nix code and finally generates an index of
@@ -10,37 +11,53 @@ set -euo pipefail
 # the maximum number of attempts before giving up inside of GET and prefetch_github
 readonly maxAttempts=30
 
-GET() {
-  local url=$1
-  local retry=1
-  echo "fetching $url" >&2
-  while ! curl -#fL -u "$GITHUB_AUTH" "$url"; do
-    echo "The curl command has failed. Attempt $retry/${maxAttempts}" >&2
-    if [[ "${retry}" -eq "${maxAttempts}" ]]; then
-      exit 1
-    fi
-    retry=$(( retry + 1 ))
-    sleep 5
-  done
-}
-
-get_org_repos() {
+get_tf_providers_org() {
+  # returns all terraform providers in a given organization, and their the
+  # latest tags, in the format
+  # $org/$repo $rev
   local org=$1
-  local page=1
-  GET "https://api.github.com/orgs/$org/repos?per_page=100" | jq -r '.[].name'
+  hub api --paginate graphql -f query="
+    query(\$endCursor: String) {
+      repositoryOwner(login: \"${org}\") {
+        repositories(first: 100, after: \$endCursor) {
+          nodes {
+            nameWithOwner
+            name
+            refs(first: 1, refPrefix: \"refs/tags/\", orderBy: {field: TAG_COMMIT_DATE, direction: DESC}) {
+              nodes {
+                name
+              }
+            }
+          }
+          pageInfo {
+            hasNextPage
+            endCursor
+          }
+        }
+      }
+  }" | \
+  jq -r '.data.repositoryOwner.repositories.nodes[] | select(.name | startswith("terraform-provider-")) | select((.refs.nodes | length) > 0) | .nameWithOwner + " " + .refs.nodes[0].name'
+  # filter the result with jq:
+  # - repos need to start with `teraform-provider-`
+  # - they need to have at least one tag
+  # for each of the remaining repos, assemble a string $org/$repo $rev
 }
 
-get_repo_tags() {
+get_latest_repo_tag() {
+  # of a given repo and owner, retrieve the latest tag
   local owner=$1
   local repo=$2
-  GET "https://api.github.com/repos/$owner/$repo/git/refs/tags?per_page=100" | \
+  hub api --paginate "https://api.github.com/repos/$owner/$repo/git/refs/tags" | \
     jq -r '.[].ref' | \
     grep -v 'v\.' | \
     cut -d '/' -f 3- | \
-    sort --version-sort
+    sort --version-sort | \
+    tail -1
 }
 
 prefetch_github() {
+  # of a given owner, repo and rev, fetch the tarball and return the output of
+  # `nix-prefetch-url`
   local owner=$1
   local repo=$2
   local rev=$3
@@ -59,7 +76,7 @@ echo_entry() {
   local owner=$1
   local repo=$2
   local rev=$3
-  local version=$(echo $3 | sed 's/^v//')
+  local version=${rev#v}
   local sha256=$4
   cat <<EOF
 {
@@ -74,69 +91,80 @@ EOF
 
 indent() { sed 's/^/    /'; }
 
-add_repo() {
+add_provider() {
   org="${1}"
   repo="${2}"
+  rev="${3}"
 
-  echo "*** $repo ***"
+  echo "*** $org/$repo $rev ***"
   name=$(echo "$repo" | cut -d - -f 3-)
-  last_tag=$(get_repo_tags "$org" "$repo" | tail -1)
-  last_tag_sha256=$(prefetch_github "$org" "$repo" "$last_tag")
+  sha256=$(prefetch_github "$org" "$repo" "$rev")
 
   {
     echo "  $name ="
-    echo_entry "$org" "$repo" "$last_tag" "$last_tag_sha256" | indent
+    echo_entry "$org" "$repo" "$rev" "$sha256" | indent
   } >> data.nix
 }
 
 ## Main ##
 
 cd "$(dirname "$0")"
 
-if [[ -z "${GITHUB_AUTH:-}" ]]; then
-  cat <<'HELP'
-Missing the GITHUB_AUTH env. This is required to work around the 60 request
-per hour rate-limit.
-
-Go to https://github.com/settings/tokens and create a new token with the
-"public_repo" scope.
-
-Then `export GITHUB_AUTH=<your user>:<your token>` and run this script again.
-HELP
-  exit 1
-fi
+# individual repos to fetch
+slugs=(
+  ajbosco/terraform-provider-segment
+  camptocamp/terraform-provider-pass
+  poseidon/terraform-provider-matchbox
+  spaceapegames/terraform-provider-wavefront
+  tweag/terraform-provider-nixos
+  tweag/terraform-provider-secret
+)
+
+# a list of providers to ignore
+blacklist=(
+  terraform-providers/terraform-provider-azure-classic
+  terraform-providers/terraform-provider-cidr
+  terraform-providers/terraform-provider-quorum
+  terraform-providers/terraform-provider-cloudinit
+  terraform-providers/terraform-provider-time
+)
 
 cat <<HEADER > data.nix
 # Generated with ./update-all
 {
 HEADER
 
-while read line; do
-  IFS=' ' read -r -a fields <<< "$line"
-  if [[ "${#fields[@]}" -eq 0 ]]; then
-    continue
-  fi
-
-  if [[ "${fields[0]}" = *"/"* ]]; then
-    org="$(echo "${fields[0]}" | cut -d/ -f1)"
-    repo="$(echo "${fields[0]}" | cut -d/ -f2)"
-    add_repo "${org}" "${repo}"
-  else
-    org="${fields[0]}"
-    repos=$(get_org_repos "$org")
-    if [[ "${#fields[@]}" -ge 2 ]] && [[ -n "${fields[1]}" ]]; then
-      repos="$( echo "${repos[@]}" | grep "${fields[1]}" )"
-    fi
-    if [[ "${#fields[@]}" -eq 3 ]] && [[ -n "${fields[2]}" ]]; then
-      repos="$( echo "${repos[@]}" | grep -v "${fields[2]}" )"
-    fi
-    repos="$( echo "${repos[@]}" | sort )"
-
-    for repo in $repos; do
-      add_repo "$org" "$repo"
-    done
-  fi
-done < <(grep -v '^#\|^$' providers.txt)
+# assemble list of terraform providers
+providers=$(get_tf_providers_org "terraform-providers")
+providers=$(echo "$providers";get_tf_providers_org "hashicorp")
+
+# add terraform-providers from slugs
+for slug in "${slugs[@]}"; do
+  # retrieve latest tag
+  org=${slug%/*}
+  repo=${slug#*/}
+  rev=$(get_latest_repo_tag "$org" "$repo")
+
+  # add to list
+  providers=$(echo "$providers";echo "$org/$repo $rev")
+done
+
+# filter out all providers on the blacklist
+for repo in "${blacklist[@]}"; do
+  providers=$(echo "$providers" | grep -v "^${repo} ")
+done
+
+# sort results alphabetically by repo name
+providers=$(echo "$providers" | sort -t "/" --key=2)
+
+# render list
+IFS=$'\n'
+for provider in $providers; do
+  org=$(echo "$provider" | cut -d " " -f 1 | cut -d "/" -f1)
+  repo=$(echo "$provider" | cut -d " " -f 1 | cut -d "/" -f2)
+  rev=$(echo "$provider" | cut -d " " -f 2)
+  add_provider "${org}" "${repo}" "${rev}"
+done
 
 cat <<FOOTER >> data.nix
 }