diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go
index d1875f31a9f..82defd77661 100644
--- a/pkg/runner/runner_test.go
+++ b/pkg/runner/runner_test.go
@@ -196,11 +196,9 @@ func (j *TestJobFileInfo) runTest(ctx context.Context, t *testing.T, cfg *Config
 	assert.Nil(t, err, j.workflowPath)
 
 	planner, err := model.NewWorkflowPlanner(fullWorkflowPath, true)
-	if err != nil {
+	if j.errorMessage != "" && err != nil {
 		assert.Error(t, err, j.errorMessage)
-	} else {
-		assert.Nil(t, err, fullWorkflowPath)
-
+	} else if assert.Nil(t, err, fullWorkflowPath) {
 		plan, err := planner.PlanEvent(j.eventName)
 		assert.True(t, (err == nil) != (plan == nil), "PlanEvent should return either a plan or an error")
 		if err == nil && plan != nil {
diff --git a/pkg/runner/testdata/actions-environment-and-context-tests/push.yml b/pkg/runner/testdata/actions-environment-and-context-tests/push.yml
index 1d799d57245..1770c55082d 100644
--- a/pkg/runner/testdata/actions-environment-and-context-tests/push.yml
+++ b/pkg/runner/testdata/actions-environment-and-context-tests/push.yml
@@ -1,5 +1,5 @@
 name: actions-with-environment-and-context-tests
-description: "Actions with environment (env vars) and context (expression) tests"
+run-name: "Actions with environment (env vars) and context (expression) tests"
 on: push
 
 jobs:
diff --git a/pkg/runner/testdata/inputs-via-env-context/action.yml b/pkg/runner/testdata/inputs-via-env-context/action/action.yml
similarity index 100%
rename from pkg/runner/testdata/inputs-via-env-context/action.yml
rename to pkg/runner/testdata/inputs-via-env-context/action/action.yml
diff --git a/pkg/runner/testdata/inputs-via-env-context/push.yml b/pkg/runner/testdata/inputs-via-env-context/push.yml
index 07fadeb1369..6f4fd8f16b0 100644
--- a/pkg/runner/testdata/inputs-via-env-context/push.yml
+++ b/pkg/runner/testdata/inputs-via-env-context/push.yml
@@ -4,7 +4,7 @@ jobs:
     runs-on: self-hosted
     steps:
     - uses: actions/checkout@v3
-    - uses: ./inputs-via-env-context
+    - uses: ./inputs-via-env-context/action
       with:
         test-env-input: ${{ env.test-env-input }}
       env:
diff --git a/pkg/runner/testdata/path-handling/action.yml b/pkg/runner/testdata/path-handling/action/action.yml
similarity index 100%
rename from pkg/runner/testdata/path-handling/action.yml
rename to pkg/runner/testdata/path-handling/action/action.yml
diff --git a/pkg/runner/testdata/path-handling/push.yml b/pkg/runner/testdata/path-handling/push.yml
index 812c8b8a289..b4570c152dd 100644
--- a/pkg/runner/testdata/path-handling/push.yml
+++ b/pkg/runner/testdata/path-handling/push.yml
@@ -18,7 +18,7 @@ jobs:
           fi
 
       - id: action-with-output
-        uses: ./path-handling/
+        uses: ./path-handling/action/
 
       - name: test path (after local action)
         run: |
diff --git a/pkg/runner/testdata/shells/bash/push.yml b/pkg/runner/testdata/shells/bash/push.yml
index 49704b2b071..12b00fcad41 100644
--- a/pkg/runner/testdata/shells/bash/push.yml
+++ b/pkg/runner/testdata/shells/bash/push.yml
@@ -5,7 +5,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: bash
         run: |
           if [[ -n "$BASH" ]]; then
             echo "I'm $BASH!"
@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     container: node:16-buster-slim
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: bash
         run: |
           if [[ -n "$BASH" ]]; then
             echo "I'm $BASH!"
diff --git a/pkg/runner/testdata/shells/pwsh/push.yml b/pkg/runner/testdata/shells/pwsh/push.yml
index 25ce66b49bc..0344fa38c3c 100644
--- a/pkg/runner/testdata/shells/pwsh/push.yml
+++ b/pkg/runner/testdata/shells/pwsh/push.yml
@@ -5,14 +5,14 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: pwsh
         run: |
           $PSVersionTable
   check-container:
     runs-on: ubuntu-latest
     container: catthehacker/ubuntu:pwsh-latest
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: pwsh
         run: |
           $PSVersionTable
   check-job-default:
diff --git a/pkg/runner/testdata/shells/python/push.yml b/pkg/runner/testdata/shells/python/push.yml
index 6480e6f0e2f..6bc3fbc3d1f 100644
--- a/pkg/runner/testdata/shells/python/push.yml
+++ b/pkg/runner/testdata/shells/python/push.yml
@@ -5,7 +5,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: python
         run: |
           import platform
           print(platform.python_version())
@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     container: node:16-buster
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: python
         run: |
           import platform
           print(platform.python_version())
diff --git a/pkg/runner/testdata/shells/sh/push.yml b/pkg/runner/testdata/shells/sh/push.yml
index 0914ca2ff28..c816326c8bc 100644
--- a/pkg/runner/testdata/shells/sh/push.yml
+++ b/pkg/runner/testdata/shells/sh/push.yml
@@ -5,7 +5,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: sh
         run: |
           if [ -z ${BASH+x} ]; then
             echo "I'm sh!"
@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     container: alpine:latest
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: sh
         run: |
           if [ -z ${BASH+x} ]; then
             echo "I'm sh!"
diff --git a/pkg/runner/testdata/windows-add-env/action.yml b/pkg/runner/testdata/windows-add-env/action/action.yml
similarity index 100%
rename from pkg/runner/testdata/windows-add-env/action.yml
rename to pkg/runner/testdata/windows-add-env/action/action.yml
diff --git a/pkg/runner/testdata/windows-add-env/push.yml b/pkg/runner/testdata/windows-add-env/push.yml
index 7d6a81df43d..d8af71525ea 100644
--- a/pkg/runner/testdata/windows-add-env/push.yml
+++ b/pkg/runner/testdata/windows-add-env/push.yml
@@ -35,7 +35,7 @@ jobs:
       env:
         KeY: 'n/a'
     - uses: actions/checkout@v3
-    - uses: ./windows-add-env
+    - uses: ./windows-add-env/action
     - name: Assert composite env is merged case insensitive
       run: exit 1
       if: env.KEY != 'n/a' || env.Key != 'n/a' || env.key != 'n/a'
\ No newline at end of file