diff --git a/.github/workflows/oidc-conformity.yml b/.github/workflows/oidc-conformity.yml
index 79fb99675..d3f1c8e7e 100644
--- a/.github/workflows/oidc-conformity.yml
+++ b/.github/workflows/oidc-conformity.yml
@@ -1,9 +1,6 @@
 name: "OpenID Connect Conformity Tests"
 
 on:
-  push:
-    branches:
-    - master
   pull_request:
     branches:
     - master
@@ -22,7 +19,9 @@ jobs:
         with:
           go-version: '^1.15.0'
       - name: Update fosite
-        run: go get github.com/${{ github.repository }}@${{ github.event.pull_request.head.sha }}
+        run: |
+          go mod edit -replace github.com/ory/fosite=github.com/${{ github.event.pull_request.head.repo.full_name }}@${{ github.event.pull_request.head.sha }}
+          go mod tidy
       - name: Start service
         run: ./test/conformance/start.sh
       - name: Run tests
diff --git a/Makefile b/Makefile
index 72a35400b..06218b8cd 100644
--- a/Makefile
+++ b/Makefile
@@ -3,3 +3,4 @@ format:
 
 test:
 		go test ./...
+