diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5fb2dc2..1a6043c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -14,7 +14,7 @@ "ghcr.io/devcontainers/features/github-cli:1": {} }, - "postCreateCommand": "sudo chsh vscode -s \"$(which pwsh)\"", + "postCreateCommand": "sudo chsh vscode -s \"$(which pwsh)\"; ./.devcontainer/postCreateCommand.ps1", // Configure tool-specific properties. "customizations": { diff --git a/.devcontainer/postCreateCommand.ps1 b/.devcontainer/postCreateCommand.ps1 new file mode 100755 index 0000000..66c8906 --- /dev/null +++ b/.devcontainer/postCreateCommand.ps1 @@ -0,0 +1,5 @@ +#!pwsh + +# install the needed pwsh modules +Install-Module Pester +Install-Module powershell-yaml \ No newline at end of file diff --git a/.github/workflows/publishing.yml b/.github/workflows/publishing.yml index 13f519f..51f419b 100644 --- a/.github/workflows/publishing.yml +++ b/.github/workflows/publishing.yml @@ -21,10 +21,11 @@ jobs: with: PAT: ${{ secrets.GITHUB_TOKEN }} id: load-actions + - shell: pwsh run: | - Write-Host "Found actions [${{ steps.load-actions.outputs.actions }}]" - $content = ${{ steps.load-actions.outputs.actions }} + Write-Host "Found actions [${{ steps.load-actions.outputs.actions-file }}]" + $content = ${{ steps.load-actions.outputs.actions-file }} New-Item -Path 'actions.json' -Value $content -Force | Out-Null $actions = $content | ConvertFrom-Json if ($actions.Length -le 0) { @@ -67,6 +68,7 @@ jobs: Release ${{ github.ref }} is available now # todo: figure out how this works with an action :-) + # does not work, as you cannot set the flag to publish the action to the marketplace #- uses: actions/publish-release-asset@v2 #with: #upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 70ae2fa..4967e7d 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -9,6 +9,32 @@ on: permissions: read-all jobs: + run-pester-tests: + runs-on: ubuntu-latest + name: Run Pester tests + steps: + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + + - name: Execute tests + shell: pwsh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # execute Pester tests + + # install powershell-yaml as it is not on the runners by default + $moduleName = "powershell-yaml" + Install-Module -Name $moduleName -Force -Scope CurrentUser -AllowClobber + + # execute tests + $testResults = Invoke-Pester + if ($testResults.FailedCount -gt 0) { + throw "Tests failed" + } + else { + Write-Host "Tests passed" + } + load-all-used-actions: runs-on: ubuntu-latest name: Test on current organization @@ -21,23 +47,8 @@ jobs: PAT: ${{ secrets.GITHUB_TOKEN }} id: load-actions - - shell: pwsh - run: | - Write-Host "Found actions [${{ steps.load-actions.outputs.actions }}]" - $content = ${{ steps.load-actions.outputs.actions }} - New-Item -Path 'actions.json' -Value $content -Force | Out-Null - $actions = $content | ConvertFrom-Json - if ($actions.Length -le 0) { - Set-Content -Value "No actions found" -Path $env:GITHUB_STEP_SUMMARY - throw "No actions found" - } - else { - Write-Host "Found [$($actions.Length)] actions" - Set-Content -Value "Found [$($actions.Length)] actions" -Path $env:GITHUB_STEP_SUMMARY - } - - shell: pwsh - name: check the output file location to contain the expected content + name: Check the output file location to contain the expected content run: | # check the output file location to contain the expected content Write-Host "Got actions file location here [${{ steps.load-actions.outputs.actions-file }}]" @@ -64,7 +75,7 @@ jobs: runs-on: ubuntu-latest name: Test on different organization env: - organization: rajbos-actions + organization: rajbos-actions-demo steps: - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 @@ -75,23 +86,8 @@ jobs: organization: ${{ env.organization }} id: load-actions - - shell: pwsh - run: | - Write-Host "Found actions [${{ steps.load-actions.outputs.actions }}]" - $content = ${{ steps.load-actions.outputs.actions }} - New-Item -Path 'actions.json' -Value $content -Force | Out-Null - $actions = $content | ConvertFrom-Json - if ($actions.Length -le 0) { - Set-Content -Value "No actions found" -Path $env:GITHUB_STEP_SUMMARY - throw "No actions found" - } - else { - Write-Host "Found [$($actions.Length)] actions" - Set-Content -Value "Found [$($actions.Length)] actions" -Path $env:GITHUB_STEP_SUMMARY - } - - shell: pwsh - name: check the output file location to contain the expected content + name: Check the output file location to contain the expected content run: | # check the output file location to contain the expected content Write-Host "Got actions file location here [${{ steps.load-actions.outputs.actions-file }}]" diff --git a/README.md b/README.md index 5b44163..b13accd 100644 --- a/README.md +++ b/README.md @@ -40,13 +40,13 @@ jobs: PAT: ${{ secrets.GITHUB_TOKEN }} # use an Access Token with correct permissions to view private repos if you need to - shell: pwsh - name: Store json file - run: echo ${{ steps.load-actions.outputs.actions }} > 'actions.json' + name: Show json file + run: cat ${{ steps.load-actions.outputs.actions-file }} - name: Upload result file as artefact uses: actions/upload-artifact@v2 with: - name: actions + name: actions-file path: actions.json ``` @@ -57,7 +57,7 @@ jobs: |PAT|The Personal Access Token to use for the API calls.| ## Outputs -actions: a compressed json string with all the actions used in the workflows in the organization. The json is in the format: +actions-file: path to file containing compressed json string with all the actions used in the workflows in the organization. The json is in the format: ``` json [ "actionLink": "actions/checkout", diff --git a/Src/PowerShell/entrypoint.ps1 b/Src/PowerShell/entrypoint.ps1 index bf0aa25..939e80a 100644 --- a/Src/PowerShell/entrypoint.ps1 +++ b/Src/PowerShell/entrypoint.ps1 @@ -47,7 +47,14 @@ function main { throw } - $actions = (.\load-used-actions.ps1 -orgName $organization -PAT $PAT) + echo $pwd + ls + + # pull in the methods from load-actions: + . $PSScriptRoot\load-used-actions.ps1 -orgName $organization -PAT $PAT + + # Get all actions + $actions = LoadAllActionsFromConfiguration # write the file outside of the container so we can pick it up Write-Host "Found [$($actions.Count)] actions " diff --git a/Src/PowerShell/github-calls.ps1 b/Src/PowerShell/github-calls.ps1 index d293575..3f2a3d5 100644 --- a/Src/PowerShell/github-calls.ps1 +++ b/Src/PowerShell/github-calls.ps1 @@ -1,4 +1,3 @@ - function Get-Headers { param ( [string] $userName, @@ -311,13 +310,54 @@ function FindAllRepos { function GetRawFile { param ( [string] $url, - [string] $PAT + [string] $PAT, + [string] $userName ) - Write-Host "Loading file content from url [$($url.Replace($PAT, "****")))]" - + if ($null -eq $PAT -or $PAT.Length -eq 0) { + Write-Warning "Cannot handle empty PAT" + return "" + } + + if ($null -eq $userName -or $userName.Length -eq 0) { + Write-Warning "Cannot handle empty userName" + return "" + } + + Write-Host "GetRawFile: $url" + if ($null -eq $url) { + Write-Warning "Cannot handle empty url" + return "" + } + $index = $url.IndexOf("?token=") + $logUrl = "" + if ($index -gt 0) { + $logUrl = $url.Substring(0, $index) + } + else { + $logUrl = $url + } + $Headers = Get-Headers -userName $userName -PAT $PAT - $result = Invoke-WebRequest -Uri $url -Headers $Headers -Method Get -ErrorAction Stop | Select-Object -Expand Content + $requestResult = "" + try { + $requestResult = Invoke-WebRequest -Uri $url -Headers $Headers -Method Get -ErrorAction Stop + } + catch { + Write-Warning "Error loading file content response from url [$($logUrl)]" + Write-Warning "Error: [$_]" + return "" + } + + try { + $result = $requestResult | Select-Object -Expand Content + } + catch { + Write-Warning "Error converting file content from url [$($logUrl)]" + Write-Warning "Error: [$_]" + Write-Warning "Content: [$requestResult]" + return "" + } return $result -} \ No newline at end of file +} diff --git a/Src/PowerShell/load-used-actions.ps1 b/Src/PowerShell/load-used-actions.ps1 index 883eaa0..7e59fee 100644 --- a/Src/PowerShell/load-used-actions.ps1 +++ b/Src/PowerShell/load-used-actions.ps1 @@ -10,21 +10,6 @@ param ( # pull in central calls library . $PSScriptRoot\github-calls.ps1 -Write-Host "We're running with these parameters:" -Write-Host "- PAT.Length: [$($PAT.Length)]" -Write-Host "- orgName: [$orgName]" - -if ($null -eq $userName -or "" -eq $userName) { - $userName = $env:GITHUB_ACTOR -} - -if ($userName -eq "dependabot[bot]") { - # try to prevent issues with [] in the username - $userName = "dependabot" -} - -Write-Host "- userName: [$userName]" -Write-Host "- marketplaceRepo: [$marketplaceRepo]" function GetActionsFromWorkflow { param ( @@ -45,51 +30,72 @@ function GetActionsFromWorkflow { Write-Warning "" Write-Warning "Error:" Write-Warning $_ - return + return $actions } - # create hastable + # create hashtable $actions = @() - - # go through the parsed yaml - foreach ($job in $parsedYaml["jobs"].GetEnumerator()) { - Write-Host " Job found: [$($job.Key)]" - $steps=$job.Value.Item("steps") - if ($null -ne $steps) { - foreach ($step in $steps) { - $uses=$step.Item("uses") - if ($null -ne $uses) { - Write-Host " Found action used: [$uses]" - $actionLink = $uses.Split("@")[0] - - $data = [PSCustomObject]@{ - actionLink = $actionLink - workflowFileName = $workflowFileName - repo = $repo - type = "action" + try { + if ($null -ne $parsedYaml["jobs"] -And "" -ne $parsedYaml["jobs"]) { #else: write info to summary? + # go through the parsed yaml + foreach ($job in $parsedYaml["jobs"].GetEnumerator()) { + Write-Host " Job found: [$($job.Key)]" + $steps=$job.Value.Item("steps") + if ($null -ne $steps) { + foreach ($step in $steps) { + $uses=$step.Item("uses") + if ($null -ne $uses) { + Write-Host " Found action used: [$uses]" + $actionLink = $uses.Split("@")[0] + + $data = [PSCustomObject]@{ + actionLink = $actionLink + workflowFileName = $workflowFileName + repo = $repo + type = "action" + } + + $actions += $data + } + } + } + else { + # check for reusable workflow + $uses = $job.Value.Item("uses") + if ($null -ne $uses) { + Write-Host " Found reusable workflow used: [$uses]" + $actionLink = $uses.Split("@")[0] + + $data = [PSCustomObject]@{ + actionLink = $actionLink + workflowFileName = $workflowFileName + repo = $repo + type = "reusable workflow" + } + + $actions += $data } - - $actions += $data } } } - else { - # check for reusable workflow - $uses = $job.Value.Item("uses") - if ($null -ne $uses) { - Write-Host " Found reusable workflow used: [$uses]" - $actionLink = $uses.Split("@")[0] - - $data = [PSCustomObject]@{ - actionLink = $actionLink - workflowFileName = $workflowFileName - repo = $repo - type = "reusable workflow" - } - - $actions += $data + } + catch { + Write-Warning "Error handling this workflow file [$($workflowFile.name)] in repo [$repo] after parsing it" + Write-Host "Error: [$_]" + Write-Host "$parsedYaml" + + if ($null -ne $env:GITHUB_STEP_SUMMARY) { + $filename = "$env:GITHUB_STEP_SUMMARY" + $content = Get-Content $filename + if ($null -eq $content -Or "" -eq $content) { + Add-Content -path $filename "# Error handling workflow file(s)" + Add-Content -path $filename "|Repository|Workflow file|Error|" + Add-Content -path $filename "|---|---|---|" } + + Add-Content -path $filename "| $repo | $($workflowFile.name) | $_ |" } + } return $actions @@ -116,7 +122,7 @@ function GetAllUsedActionsFromRepo { foreach ($workflowFile in $workflowFiles) { try { if ($null -ne $workflowFile.download_url -and $workflowFile.download_url.Length -gt 0 -and $workflowFile.download_url.Split("?")[0].EndsWith(".yml")) { - $workflow = GetRawFile -url $workflowFile.download_url -PAT $PAT + $workflow = GetRawFile -url $workflowFile.download_url -PAT $PAT -userName $userName $actions = GetActionsFromWorkflow -workflow $workflow -workflowFileName $workflowFile.name -repo $repo $actionsInRepo += $actions @@ -128,7 +134,7 @@ function GetAllUsedActionsFromRepo { Write-Warning "----------------------------------" Write-Host "Error: [$_]" Write-Warning "----------------------------------" - continue + #continue } } @@ -205,7 +211,23 @@ function LoadAllUsedActionsFromRepos { return $actions } -function main() { +function LoadAllActionsFromConfiguration() { + + Write-Host "We're running with these parameters:" + Write-Host "- PAT.Length: [$($PAT.Length)]" + Write-Host "- orgName: [$orgName]" + + if ($null -eq $userName -or "" -eq $userName) { + $userName = $env:GITHUB_ACTOR + } + + if ($userName -eq "dependabot[bot]") { + # try to prevent issues with [] in the username + $userName = "dependabot" + } + + Write-Host "- userName: [$userName]" + Write-Host "- marketplaceRepo: [$marketplaceRepo]" # get all repos in an org $repos = FindAllRepos -orgName $orgName -userName $userName -PAT $PAT @@ -234,6 +256,3 @@ function main() { return $summarizeActions } - -$actions = main -return $actions diff --git a/Tests/ConvertFrom-Yaml.Tests.ps1 b/Tests/ConvertFrom-Yaml.Tests.ps1 new file mode 100644 index 0000000..d6e9bb2 --- /dev/null +++ b/Tests/ConvertFrom-Yaml.Tests.ps1 @@ -0,0 +1,65 @@ +BeforeAll { + Import-Module "powershell-yaml" -Force +} + +Describe "Test conversion with multiple indentation" { + It "Extra indentations" { + + Write-Host $PSScriptRoot + $content = Get-Content "Tests/Files/extra-indentation.yml" -Raw + $result = ConvertFrom-Yaml $content + + $result["jobs"].GetEnumerator().Length | Should -Be 1 + foreach ($job in $result["jobs"].GetEnumerator()) { + $stepLength = 0 + $steps = $job.Value.Item("steps") + foreach ($step in $steps) { + $stepLength++ + } + $stepLength | Should -Be 3 + } + } +} + +Describe "Test conversion with normal indentation" { + It "Normal indentations" { + + Write-Host $PSScriptRoot + $content = Get-Content "Tests/Files/normal-indentation.yml" -Raw + $result = ConvertFrom-Yaml $content + + $result["jobs"].GetEnumerator().Length | Should -Be 1 + foreach ($job in $result["jobs"].GetEnumerator()) { + $stepLength = 0 + $steps = $job.Value.Item("steps") + foreach ($step in $steps) { + $stepLength++ + } + $stepLength | Should -Be 3 + } + } + + It "Slim indentation" { + Write-Host $PSScriptRoot + $content = Get-Content "Tests/Files/rajbos-actions-demo_deploy-cloudrun.yml" -Raw + $result = ConvertFrom-Yaml $content + + $jobCount = 0 + foreach ($job in $result["jobs"].GetEnumerator()) { + $jobCount++ + + $stepLength = 0 + $steps = $job.Value.Item("steps") + foreach ($step in $steps) { + $stepLength++ + } + switch ($job.Key) { # jobs are not ordered as in the file + "gcloud" { $stepLength | Should -Be 7 -Because "$($job.Key) should have 5 steps"} + "b64_json" { $stepLength | Should -Be 7 -Because "$($job.Key) should have 7 steps" } + "json" { $stepLength | Should -Be 7 -Because "$($job.Key) should have 7 steps" } + "cleanup" { $stepLength | Should -Be 2 -Because "$($job.Key) should have 2 steps" } + } + } + $jobCount | Should -Be 4 + } +} \ No newline at end of file diff --git a/Tests/Files/extra-indentation.yml b/Tests/Files/extra-indentation.yml new file mode 100644 index 0000000..dbc7e4b --- /dev/null +++ b/Tests/Files/extra-indentation.yml @@ -0,0 +1,17 @@ +name: Build +on: + push: + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + steps: + - name: Get source code + uses: actions/checkout@v2 + - name: Restore depdencies + run: npm ci + - name: Build + run: npm run all \ No newline at end of file diff --git a/Tests/Files/normal-indentation.yml b/Tests/Files/normal-indentation.yml new file mode 100644 index 0000000..6e8f9a8 --- /dev/null +++ b/Tests/Files/normal-indentation.yml @@ -0,0 +1,17 @@ +name: Build +on: + push: + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + steps: + - name: Get source code + uses: actions/checkout@v2 + - name: Restore depdencies + run: npm ci + - name: Build + run: npm run all \ No newline at end of file diff --git a/Tests/Files/rajbos-actions-demo_deploy-cloudrun.yml b/Tests/Files/rajbos-actions-demo_deploy-cloudrun.yml new file mode 100644 index 0000000..2ad892c --- /dev/null +++ b/Tests/Files/rajbos-actions-demo_deploy-cloudrun.yml @@ -0,0 +1,123 @@ +# from https://raw.githubusercontent.com/rajbos-actions-demo/deploy-cloudrun/main/.github/workflows/deploy-cloudrun-credentials-it.yml +name: deploy-cloudrun Credentials Integration + +on: + push: + branches-ignore: + - 'example-*' + +jobs: + gcloud: + name: with setup-gcloud + if: ${{ github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name }} + runs-on: ubuntu-latest + steps: + - id: service + run: echo ::set-output name=service::run-gcloud-$GITHUB_SHA + - uses: actions/checkout@v2 + - uses: actions/setup-node@master + with: + node-version: 12.x + - run: |- + npm install + npm run build + + - name: Set up authentication + uses: google-github-actions/setup-gcloud@master + with: + service_account_key: ${{ secrets.DEPLOY_CLOUDRUN_SA_KEY_B64 }} + export_default_credentials: true + + - id: deploy + uses: ./ + with: + image: gcr.io/cloudrun/hello + service: ${{ steps.service.outputs.service }} + + - name: Integration Tests + run: npm run e2e-tests + env: + URL: ${{ steps.deploy.outputs.url }} + + b64_json: + name: with base64 creds + if: ${{ github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name }} + runs-on: ubuntu-latest + steps: + - id: service + run: echo ::set-output name=service::run-b64-$GITHUB_SHA + - uses: actions/checkout@v2 + - uses: actions/setup-node@master + with: + node-version: 12.x + - run: |- + npm install + npm run build + + - id: deploy + uses: ./ + with: + credentials: ${{ secrets.DEPLOY_CLOUDRUN_SA_KEY_B64 }} + image: gcr.io/cloudrun/hello + service: ${{ steps.service.outputs.service }} + + - uses: google-github-actions/setup-gcloud@master # Set up ADC to make authenticated request to service + with: + service_account_key: ${{ secrets.DEPLOY_CLOUDRUN_SA_KEY_B64 }} + export_default_credentials: true + + - name: Integration Tests + run: npm run e2e-tests + env: + URL: ${{ steps.deploy.outputs.url }} + + json: + name: with json creds + if: ${{ github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name }} + runs-on: ubuntu-latest + steps: + - id: service + run: echo ::set-output name=service::run-json-$GITHUB_SHA + - uses: actions/checkout@v2 + - uses: actions/setup-node@master + with: + node-version: 12.x + - run: |- + npm install + npm run build + + - id: deploy + uses: ./ + with: + credentials: ${{ secrets.DEPLOY_CLOUDRUN_SA_KEY_JSON }} + image: gcr.io/cloudrun/hello + service: ${{ steps.service.outputs.service }} + + - uses: google-github-actions/setup-gcloud@master # Set up ADC to make authenticated request to service + with: + service_account_key: ${{ secrets.DEPLOY_CLOUDRUN_SA_KEY_B64 }} + export_default_credentials: true + + - name: Integration Tests + run: npm run e2e-tests + env: + URL: ${{ steps.deploy.outputs.url }} + + cleanup: + name: Clean Up + if: ${{ always() }} + runs-on: ubuntu-latest + needs: [json, gcloud, b64_json] + steps: + - uses: google-github-actions/setup-gcloud@master + with: + service_account_key: ${{ secrets.DEPLOY_CLOUDRUN_SA_KEY_B64 }} + project_id: ${{ secrets.DEPLOY_CLOUDRUN_PROJECT_ID }} + + - name: Delete services + run: |- + gcloud config set run/platform managed + gcloud config set run/region us-central1 + gcloud run services delete run-json-$GITHUB_SHA --quiet + gcloud run services delete run-b64-$GITHUB_SHA --quiet + gcloud run services delete run-gcloud-$GITHUB_SHA --quiet \ No newline at end of file diff --git a/Tests/Integration.Tests.ps1 b/Tests/Integration.Tests.ps1 new file mode 100644 index 0000000..204475d --- /dev/null +++ b/Tests/Integration.Tests.ps1 @@ -0,0 +1,40 @@ +# set up environment variables + +$Global:PAT +$Global:userName + +BeforeAll { + $Global:PAT = $env:GITHUB_TOKEN + $Global:userName = "rajbos" + + Import-Module "powershell-yaml" -Force + # pull in central calls library + . .\Src\PowerShell\github-calls.ps1 + . .\Src\PowerShell\load-used-actions.ps1 -orgName "rajbos-actions" -userName "rajbos" -marketplaceRepo "rajbos/actions-marketplace" -PAT $Global:PAT +} + +Describe "Download OSSF workflow" { + It "Parse yaml" { + $url = "https://raw.githubusercontent.com/devops-actions/load-used-actions/main/.github/workflows/ossf-analysis.yml" + + $workflow = GetRawFile -url $url -PAT $Global:PAT -userName $Global:userName + + $workflow | Should -Not -BeNullOrEmpty + $workflow.Replace(" ", "") | Should -Not -BeNullOrEmpty + + } +} + +Describe "Download Cloudrun workflow" { + It "Parse yaml" { + $url = "https://raw.githubusercontent.com/rajbos-actions-demo/deploy-cloudrun/main/.github/workflows/deploy-cloudrun.yml" + + $workflow = GetRawFile -url $url -PAT $Global:PAT -userName $Global:userName + + $workflow | Should -Not -BeNullOrEmpty + $workflow.Replace(" ", "") | Should -Not -BeNullOrEmpty + $actions = GetActionsFromWorkflow -workflow $workflow -workflowFileName "deploy-cloudrun.yml" -repo "rajbos-actions-demo/deploy-cloudrun" + $actions | Should -Not -BeNullOrEmpty + $actions.Count | Should -Be 3 + } +} \ No newline at end of file diff --git a/action.yml b/action.yml index 17535d1..4a3609c 100644 --- a/action.yml +++ b/action.yml @@ -13,8 +13,6 @@ inputs: description: 'Personal access token to use for analysis. Leaving this empty will use the default GITHUB_TOKEN that could have limited access rights' required: true outputs: - actions: - description: 'List of all actions used in the organization' actions-file: description: 'Location of the file containing the list of all actions used in the organization' runs: