all: migrate the rest of the CircleCI jobs to GitHub Actions

.github/actions/setup-go/action.yml

@@ -18,7 +18,7 @@ runs:
     shell: bash
     run: |
       # Set a base, fixed modification time of all directories.
-      # git-restore-mtime doesn't set the mtime of of all directories.
+      # git-restore-mtime doesn't set the mtime of all directories.
       # (see https://github.com/MestreLion/git-tools/issues/47 for details)
       touch -m -t '201509301646' $(find . -type d -not -path '.git/*')
       # Restore original modification time from git. git clone sets the
@@ -42,7 +42,7 @@ runs:
   # rebuilds and cause test runs to skip tests that have no reason to rerun.
   # Don't run this for protected branches like master/main.
   - uses: actions/cache@v2
-    if: ${{ github.ref_protected || github.ref != 'master' || github.ref != 'main' }}
+    if: github.ref_protected || github.ref != 'refs/heads/master'
     with:
       path: ~/.cache/go-build
       key: ${{ env.PREFIX }}-go-build-${{ github.ref }}-${{ hashFiles('**', '!.git') }}

.github/workflows/go.yml

@@ -32,6 +32,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v2
+      with:
+        # For pull requests, make sure we checkout `ref/pull/<PR_number>/head`
+        # and not `ref/pull/<PR_number>/merge` (which doesn't exist for the PR creator and
+        # is what github.ref points to)
+        ref: ${{ github.head_ref || github.ref }}
     - uses: ./.github/actions/setup-go
       with:
         go-version: ${{ matrix.go }}
@@ -78,6 +83,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
       with:
+        # For pull requests, make sure we checkout `ref/pull/<PR_number>/head`
+        # and not `ref/pull/<PR_number>/merge` (which doesn't exist for the PR creator and
+        # is what github.ref points to)
+        ref: ${{ github.head_ref || github.ref }}
         # We need to full history for git-restore-mtime to know what modification dates to use.
         # Otherwise, the Go test cache will fail (due to the modification time of fixtures changing).
         fetch-depth: '0'

.github/workflows/horizon-master.yml

@@ -0,0 +1,54 @@
+name: Horizon master
+
+on:
+  push:
+    branches: [master]
+
+jobs:
+
+  push-horizon-image-sha:
+    name: Push stellar/horizon:sha to DockerHub
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Get image tag (short sha)
+        shell: bash
+        id: get_tag
+        run: echo ::set-output name=TAG::$(git rev-parse --short ${{ github.sha }} )
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USER }}
+          password: ${{ secrets.DOCKER_PASS }}
+
+      - name: Build and push to DockerHub
+        uses: docker/build-push-action@v2
+        with:
+          # TODO: Commented out until we disable the CircleCI jobs
+          # push: true
+          tags: stellar/horizon:${{ steps.get_tag.outputs.TAG }}
+          file: services/horizon/docker/Dockerfile.dev
+
+  push-state-diff-image:
+    name: Push stellar/ledger-state-diff:{sha,latest} to DockerHub
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USER }}
+          password: ${{ secrets.DOCKER_PASS }}
+
+      - name: Build and push to DockerHub
+        uses: docker/build-push-action@v2
+        with:
+          # TODO: Commented out until we disable the CircleCI jobs
+          # push: true
+          tags: stellar/ledger-state-diff:${{ github.sha }},stellar/ledger-state-diff:latest
+          file: exp/tools/dump-ledger-state/Dockerfile
+          build-args: GITCOMMIT=${{ github.sha }}
+          no-cache: true

.github/workflows/horizon-release.yml

@@ -0,0 +1,71 @@
+name: Horizon release
+
+on:
+  push:
+    tags: ['horizon-v*']
+
+jobs:
+
+  publish-artifacts:
+    runs-on: ubuntu-latest
+    name: Upload artifacts to GitHub release
+    steps:
+      - name: Run deprecation tests
+        shell: bash
+        run: |
+          export VERSION=${GITHUB_REF_NAME#horizon-v}
+          echo "Searching for \"Action needed in release ${VERSION}\" tags..."
+          # Negate the result so process exits with 1 if anything found
+          ! egrep -irn -A 1 --include=*.go "Action.+needed.+in.+release:.+$VERSION" ./
+
+      - uses: actions/checkout@v2
+
+      - uses: ./.github/actions/setup-go
+        with:
+          go-version: 1.17
+
+      - name: Check dependencies
+        run: ./gomod.sh
+
+      - name: Build binaries
+        run: go run ./support/scripts/build_release_artifacts/main.go
+
+      # TODO: Disabled until we remove the CircleCI jobs
+      - if: false
+        name: Upload binaries to release
+        uses: svenstaro/upload-release-action@483c1e56f95e88835747b1c7c60581215016cbf2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ github.ref_name }}
+          file: ./dist/*
+          file_glob: true
+          overwrite: true
+
+  push-horizon-image:
+    runs-on: ubuntu-latest
+    name: Push stellar/horizon:{version,latest} to DockerHub
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Extract Horizon version
+        shell: bash
+        id: get_version
+        run: echo ::set-output name=VERSION::${GITHUB_REF_NAME#horizon-v}
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USER }}
+          password: ${{ secrets.DOCKER_PASS }}
+
+      - name: Build and push to DockerHub
+        uses: docker/build-push-action@v2
+        with:
+          # TODO: Commented out until we disable the CircleCI jobs
+          # push: true
+          # TODO(fons): pushing stellar/horizon:latest should be reviewed and manually approved.
+          # We might not want the latest tag here being pushed on every release.
+          # We might release a patch for older versions of horizon.
+          # Otherwise, if we are currently on e.g. v1.6.1 and need to release v1.5.3, v1.5.3 will become latest.
+          tags: stellar/horizon:latest,stellar/horizon:${{ steps.get_version.outputs.VERSION }}
+          file: services/horizon/docker/Dockerfile.dev

.github/workflows/horizon.yml

@@ -8,6 +8,7 @@ on:
 jobs:
 
   integration:
+    name: Integration tests
     strategy:
       matrix:
         os: [ubuntu-latest]
@@ -40,14 +41,21 @@ jobs:
     steps:
     - uses: actions/checkout@v2
       with:
+        # For pull requests, make sure we checkout `ref/pull/<PR_number>/head`
+        # and not `ref/pull/<PR_number>/merge` (which doesn't exist for the PR creator and
+        # is what github.ref points to)
+        ref: ${{ github.head_ref || github.ref }}
         # We need to full history for git-restore-mtime to know what modification dates to use.
         # Otherwise, the Go test cache will fail (due to the modification time of fixtures changing).
         fetch-depth: '0'
+
     - uses: ./.github/actions/setup-go
       with:
         go-version: ${{ matrix.go }}
+
     - name: Pull latest Stellar Core image
       run: docker pull stellar/stellar-core
+
     - if: ${{ startsWith(matrix.ingestion-backend, 'captive-core') }}
       name: Install and enable Captive Core
       run: |
@@ -57,8 +65,39 @@ jobs:
         echo "Using stellar core version $(stellar-core version)"
         echo 'HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE=true' >> $GITHUB_ENV
         echo 'CAPTIVE_CORE_BIN=/usr/bin/stellar-core'       >> $GITHUB_ENV
+
     - if: ${{ matrix.ingestion-backend == 'captive-core-remote-storage' }}
       name: Setup Captive Core Remote Storage
       run: echo 'HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE_USE_DB=true' >> $GITHUB_ENV
+
     - run: go test -race -timeout 25m -v ./services/horizon/internal/integration/...
 
+  verify-range:
+    name: Test (and push) verify-range image
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        # For pull requests, make sure we checkout `ref/pull/<PR_number>/head`
+        # and not `ref/pull/<PR_number>/merge` (which doesn't exist for the PR creator and
+        # is what github.ref points to)
+        ref: ${{ github.head_ref || github.ref }}
+
+    - name: Build and test the Verify Range Docker image
+      run: |
+        docker build -f services/horizon/docker/verify-range/Dockerfile -t stellar/horizon-verify-range services/horizon/docker/verify-range/
+        # Any range should do for basic testing, this range was chosen pretty early in history so that it only takes a few mins to run
+        docker run -e BRANCH=$(git rev-parse HEAD) -e FROM=10000063 -e TO=10000127 stellar/horizon-verify-range
+
+    # Push image
+    - if: github.ref == 'refs/heads/master'
+      name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_USER }}
+        password: ${{ secrets.DOCKER_PASS }}
+    - if: github.ref == 'refs/heads/master'
+      name: Push to DockerHub
+      # TODO(fons): Commented out until we disable the CircleCI jobs
+      run: \# docker push stellar/horizon-verify-range:latest
+

services/horizon/docker/verify-range/dependencies

@@ -11,7 +11,7 @@ echo "deb https://apt.stellar.org $(lsb_release -cs) unstable" | sudo tee -a /et
 apt-get update
 apt-get install -y stellar-core=${STELLAR_CORE_VERSION}
 
-wget https://dl.google.com/go/go1.17.linux-amd64.tar.gz
+wget -q https://dl.google.com/go/go1.17.linux-amd64.tar.gz
 tar -C /usr/local -xzf go1.17.linux-amd64.tar.gz
 
 # configure postgres

support/scripts/build_release_artifacts/main.go

@@ -43,8 +43,11 @@ func main() {
 	if os.Getenv("TRAVIS_EVENT_TYPE") == "cron" {
 		buildNightlies()
 		os.Exit(0)
-	} else if os.Getenv("CIRCLE_TAG") != "" {
-		buildByTag()
+	} else if circleTag := os.Getenv("CIRCLE_TAG"); circleTag != "" {
+		buildByTag(circleTag)
+		os.Exit(0)
+	} else if ghTag := getGitHubTagName(); ghTag != "" {
+		buildByTag(ghTag)
 		os.Exit(0)
 	} else {
 		buildSnapshots()
@@ -54,6 +57,20 @@ func main() {
 	log.Info("nothing to do")
 }
 
+func getGitHubTagName() string {
+	const githubTagRefPrefix = "refs/tags/"
+	ref := os.Getenv("GITHUB_REF")
+	if ref == "" {
+		// we are not in a github action
+		return ""
+	}
+	if !strings.HasPrefix(ref, githubTagRefPrefix) {
+		// we are not in a tag-triggered event
+		return ""
+	}
+	return strings.TrimPrefix(ref, githubTagRefPrefix)
+}
+
 // package searches the `tools` and `services` packages of this repo to find
 // the source directory.  This is used within the script to find the README and
 // other files that should be packaged with the binary.
@@ -145,16 +162,16 @@ func buildNightlies() {
 	}
 }
 
-func buildByTag() {
-	bin, version := extractFromTag(os.Getenv("CIRCLE_TAG"))
-	pkg := packageName(bin)
-	repo := repoName()
-
+func buildByTag(tag string) {
+	bin, version := extractFromTag(tag)
 	if bin == "" {
-		log.Info("could not extract info from CIRCLE_TAG: skipping artifact packaging")
+		log.Infof("non-conformant tag name %q: skipping artifact packaging", tag)
 		os.Exit(0)
 	}
 
+	pkg := packageName(bin)
+	repo := repoName()
+
 	// Don't build anything if no package can be found
 	if pkg == "" {
 		log.Infof("could not find `%s` in expected binary locations: skipping artifact packaging", bin)
@@ -207,12 +224,11 @@ func buildSnapshots() {
 
 // extractFromTag extracts the name of the binary that should be packaged in the
 // course of execution this script as well as the version it should be packaged
-// as, based on the name of the tag in the CIRCLE_TAG environment variable.
+// as, based on the name of the tag.
 // Tags must be of the form `NAME-vSEMVER`, such as `horizon-v1.0.0` to be
 // matched by this function.
 //
-// In the event that the CIRCLE_TAG is missing or the match fails, an empty
-// string will be returned.
+// In the event the match fails, an empty string will be returned.
 func extractFromTag(tag string) (string, string) {
 	match := extractBinName.FindStringSubmatch(tag)
 	if match == nil {