Automate SDK release from CI

.github/templates/Readme.md

@@ -4,9 +4,16 @@ We're using [ytt](https://github.com/vmware-tanzu/carvel-ytt) to generate the gi
 
 ## Prerequisites
 
+Get ytt from your package manager of choice. If, for example, you're on Windows and use chocolatey you can run:<br/> `choco install ytt -y`
+
+If instead ytt isn't in your packager manager or you don't use one you can manually get ytt as follows:
+
 1. Download the executable for your platform from the [releases page](https://github.com/vmware-tanzu/carvel-ytt/releases).
 1. Rename it to `ytt`.
 1. Place it in your PATH.
 
 ## Building the docs
-1. In `$SolutionDir/.github/templates` run `ytt -f main.yml --output-files ../workflows/`.
+1. `cd $SolutionDir/.github/templates`
+1. `ytt -f . --output-files ../workflows/` ==> to target all files  
+or  
+`ytt -f YOUR_FILE --output-files ../workflows/` ==> for a specific file

.github/templates/main.yml

@@ -30,11 +30,8 @@ if: #@ wrappersCacheCondition
 #@ def setupVcpkg():
 name: Setup Vcpkg
 run: |
-  Write-Output 'Beginning download...'
   Invoke-WebRequest -Uri https://static.realm.io/downloads/vcpkg.zip -OutFile C:\vcpkg.zip
-  Write-Output ((Get-Item C:\vcpkg.zip).length/1MB)
   Expand-Archive -Path C:\vcpkg.zip -DestinationPath C:\
-  Write-Output 'Completed!'
 shell: powershell
 if: #@ wrappersCacheCondition + " && steps.check-vcpkg-cache.outputs.cache-hit != 'true'"
 #@ end
@@ -50,7 +47,7 @@ if: #@ wrappersCacheCondition + " && steps.check-vcpkg-cache.outputs.cache-hit !
 #@ actualCommand = cmd + configurationParam + ltoParam
 
 steps:
-  - #@ template.replace(checkoutCode())
+  - #@ template.replace(checkoutCode("recursive"))
   - #@ checkCache(cacheKey)
   #@ for step in intermediateSteps:
   - #@ step
@@ -66,11 +63,12 @@ steps:
       retention-days: 1
 #@ end
 
-#@ def checkoutCode():
+#@ def checkoutCode(submodules=False):
   - name: Checkout code
     uses: actions/checkout@v2
     with:
-      submodules: recursive
+      submodules: #@ submodules
+      ref: ${{ github.event.pull_request.head.sha }}
   - name: Register csc problem matcher
     run: echo "::add-matcher::.github/problem-matchers/csc.json"
   - name: Register msvc problem matcher
@@ -101,6 +99,35 @@ steps:
       retention-days: ${{ github.event_name != 'pull_request' && 30 || 1 }}
 #@ end
 
+#@ def buildDocs():
+#@ docsCondition = "${{ !contains(github.ref, 'release') }}"
+  - name: Check Docfx cache
+    id: check-docfx-cache
+    if: #@ docsCondition
+    uses: actions/cache@v2
+    with:
+      path: 'C:\docfx'
+      key: docfx
+  - name: Download docfx
+    if: ${{ steps.check-docfx-cache.outputs.cache-hit != 'true' && !contains(github.ref, 'release') }}
+    run: |
+      Invoke-WebRequest -Uri https://github.com/dotnet/docfx/releases/download/v2.58/docfx.zip -OutFile C:\docfx.zip
+      Expand-Archive -Path C:\docfx.zip -DestinationPath C:\docfx
+    shell: powershell
+  - name: Build docs
+    if: #@ docsCondition
+    run: |
+      C:\docfx\docfx Docs/docfx.json
+      Compress-Archive -Path Docs/_site -DestinationPath "Realm/packages/Docs.zip"
+  - name: Store docs artifacts
+    if: #@ docsCondition
+    uses: actions/upload-artifact@v2
+    with:
+      name: Docs.zip
+      path: ${{ github.workspace }}/Realm/packages/Docs.zip
+      retention-days: ${{ github.event_name != 'pull_request' && 30 || 1 }}
+#@ end
+
 #@ def uploadPackageArtifacts():
 #@ for pkgName in nugetPackages:
 #@ finalPkgName = pkgName + ".${{ steps.find-nupkg-version.outputs.package_version }}.nupkg"
@@ -114,13 +141,13 @@ steps:
 #@ end
 
 #@ def fetchPackageArtifacts():
-#@ for pkg in [ "Realm", "Realm.Fody" ]:
+#@  for pkg in [ "Realm", "Realm.Fody" ]:
   - name: #@ "Fetch " + pkg
     uses: actions/download-artifact@v2
     with:
       name: #@ pkg + ".${{ needs.build-packages.outputs.package_version }}.nupkg"
       path: ${{ github.workspace }}/Realm/packages/
-#@ end
+#@  end
 #@ end
 
 #@ def deleteTempPackageArtifacts():
@@ -288,10 +315,24 @@ jobs:
       - name: Set version suffix
         id: set-version-suffix
         #! Build suffix is PR-1234.5 for PR builds or alpha.123 for branch builds.
-        run: echo "::set-output name=build_suffix::${{ github.event_name == 'pull_request' && format('PR-{0}', github.event.number) || 'alpha'}}.${{ github.run_number }}"
+        run: |
+          $suffix = ""
+          if ($env:GITHUB_EVENT_NAME -eq "pull_request")
+          {
+            if (-Not $env:GITHUB_REF.Contains("release"))
+            {
+              $suffix = "PR-${{ github.event.number }}.$env:GITHUB_RUN_NUMBER"
+            }
+          }
+          else
+          {
+            $suffix = "alpha.$env:GITHUB_RUN_NUMBER"
+          }
+          echo "::set-output name=build_suffix::$suffix"
       - #@ template.replace(fetchWrapperBinaries())
       - #@ template.replace(buildPackages())
       - #@ findPackageVersion()
+      - #@ template.replace(buildDocs())
       - #@ template.replace(uploadPackageArtifacts())
       - #@ template.replace(buildUnityPackage())
   run-tests-net-framework:

.github/workflows/main.yml

@@ -14,6 +14,7 @@ jobs:
       uses: actions/checkout@v2
       with:
         submodules: recursive
+        ref: ${{ github.event.pull_request.head.sha }}
     - name: Register csc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/csc.json"
     - name: Register msvc problem matcher
@@ -41,6 +42,7 @@ jobs:
       uses: actions/checkout@v2
       with:
         submodules: recursive
+        ref: ${{ github.event.pull_request.head.sha }}
     - name: Register csc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/csc.json"
     - name: Register msvc problem matcher
@@ -68,6 +70,7 @@ jobs:
       uses: actions/checkout@v2
       with:
         submodules: recursive
+        ref: ${{ github.event.pull_request.head.sha }}
     - name: Register csc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/csc.json"
     - name: Register msvc problem matcher
@@ -102,6 +105,7 @@ jobs:
       uses: actions/checkout@v2
       with:
         submodules: recursive
+        ref: ${{ github.event.pull_request.head.sha }}
     - name: Register csc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/csc.json"
     - name: Register msvc problem matcher
@@ -134,6 +138,7 @@ jobs:
       uses: actions/checkout@v2
       with:
         submodules: recursive
+        ref: ${{ github.event.pull_request.head.sha }}
     - name: Register csc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/csc.json"
     - name: Register msvc problem matcher
@@ -153,11 +158,8 @@ jobs:
       if: steps.check-cache.outputs.cache-hit != 'true'
     - name: Setup Vcpkg
       run: |
-        Write-Output 'Beginning download...'
         Invoke-WebRequest -Uri https://static.realm.io/downloads/vcpkg.zip -OutFile C:\vcpkg.zip
-        Write-Output ((Get-Item C:\vcpkg.zip).length/1MB)
         Expand-Archive -Path C:\vcpkg.zip -DestinationPath C:\
-        Write-Output 'Completed!'
       shell: powershell
       if: steps.check-cache.outputs.cache-hit != 'true' && steps.check-vcpkg-cache.outputs.cache-hit != 'true'
     - name: Build wrappers
@@ -183,6 +185,7 @@ jobs:
       uses: actions/checkout@v2
       with:
         submodules: recursive
+        ref: ${{ github.event.pull_request.head.sha }}
     - name: Register csc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/csc.json"
     - name: Register msvc problem matcher
@@ -202,11 +205,8 @@ jobs:
       if: steps.check-cache.outputs.cache-hit != 'true'
     - name: Setup Vcpkg
       run: |
-        Write-Output 'Beginning download...'
         Invoke-WebRequest -Uri https://static.realm.io/downloads/vcpkg.zip -OutFile C:\vcpkg.zip
-        Write-Output ((Get-Item C:\vcpkg.zip).length/1MB)
         Expand-Archive -Path C:\vcpkg.zip -DestinationPath C:\
-        Write-Output 'Completed!'
       shell: powershell
       if: steps.check-cache.outputs.cache-hit != 'true' && steps.check-vcpkg-cache.outputs.cache-hit != 'true'
     - name: Build wrappers
@@ -236,14 +236,28 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
       with:
-        submodules: recursive
+        submodules: false
+        ref: ${{ github.event.pull_request.head.sha }}
     - name: Register csc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/csc.json"
     - name: Register msvc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/msvc.json"
     - name: Set version suffix
       id: set-version-suffix
-      run: echo "::set-output name=build_suffix::${{ github.event_name == 'pull_request' && format('PR-{0}', github.event.number) || 'alpha'}}.${{ github.run_number }}"
+      run: |
+        $suffix = ""
+        if ($env:GITHUB_EVENT_NAME -eq "pull_request")
+        {
+          if (-Not $env:GITHUB_REF.Contains("release"))
+          {
+            $suffix = "PR-${{ github.event.number }}.$env:GITHUB_RUN_NUMBER"
+          }
+        }
+        else
+        {
+          $suffix = "alpha.$env:GITHUB_RUN_NUMBER"
+        }
+        echo "::set-output name=build_suffix::$suffix"
     - name: Fetch artifacts for macos
       uses: actions/download-artifact@v2
       with:
@@ -320,6 +334,31 @@ jobs:
         pkgName=${tmpName#"Realm.Fody."}
         echo "::set-output name=package_version::$pkgName"
       shell: bash
+    - name: Check Docfx cache
+      id: check-docfx-cache
+      if: ${{ !contains(github.ref, 'release') }}
+      uses: actions/cache@v2
+      with:
+        path: C:\docfx
+        key: docfx
+    - name: Download docfx
+      if: ${{ steps.check-docfx-cache.outputs.cache-hit != 'true' && !contains(github.ref, 'release') }}
+      run: |
+        Invoke-WebRequest -Uri https://github.com/dotnet/docfx/releases/download/v2.58/docfx.zip -OutFile C:\docfx.zip
+        Expand-Archive -Path C:\docfx.zip -DestinationPath C:\docfx
+      shell: powershell
+    - name: Build docs
+      if: ${{ !contains(github.ref, 'release') }}
+      run: |
+        C:\docfx\docfx Docs/docfx.json
+        Compress-Archive -Path Docs/_site -DestinationPath "Realm/packages/Docs.zip"
+    - name: Store docs artifacts
+      if: ${{ !contains(github.ref, 'release') }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: Docs.zip
+        path: ${{ github.workspace }}/Realm/packages/Docs.zip
+        retention-days: ${{ github.event_name != 'pull_request' && 30 || 1 }}
     - name: Store artifacts for Realm.Fody
       uses: actions/upload-artifact@v2
       with:
@@ -360,7 +399,8 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
       with:
-        submodules: recursive
+        submodules: false
+        ref: ${{ github.event.pull_request.head.sha }}
     - name: Register csc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/csc.json"
     - name: Register msvc problem matcher
@@ -396,7 +436,8 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
       with:
-        submodules: recursive
+        submodules: false
+        ref: ${{ github.event.pull_request.head.sha }}
     - name: Register csc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/csc.json"
     - name: Register msvc problem matcher
@@ -472,7 +513,8 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
       with:
-        submodules: recursive
+        submodules: false
+        ref: ${{ github.event.pull_request.head.sha }}
     - name: Register csc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/csc.json"
     - name: Register msvc problem matcher
@@ -508,7 +550,8 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
       with:
-        submodules: recursive
+        submodules: false
+        ref: ${{ github.event.pull_request.head.sha }}
     - name: Register csc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/csc.json"
     - name: Register msvc problem matcher
@@ -542,7 +585,8 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
       with:
-        submodules: recursive
+        submodules: false
+        ref: ${{ github.event.pull_request.head.sha }}
     - name: Register csc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/csc.json"
     - name: Register msvc problem matcher
@@ -580,7 +624,8 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
       with:
-        submodules: recursive
+        submodules: false
+        ref: ${{ github.event.pull_request.head.sha }}
     - name: Register csc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/csc.json"
     - name: Register msvc problem matcher

.github/workflows/release.yml

@@ -0,0 +1,59 @@
+name: release
+on:
+  workflow_dispatch
+
+jobs:
+  publish-to-nuget:
+    runs-on: windows-latest
+    name: Publish nupkg to nuget
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Download all artifacts
+      uses: dawidd6/action-download-artifact@v2
+      with:
+        workflow: main.yml
+        branch: ${{ github.ref }}
+        path: ${{ github.workspace }}/Realm/packages/
+    - name: Extract release notes
+      id: extract_release_notes
+      run: |
+        $changelogContent = Get-Content -Raw -Path "./CHANGELOG.md"
+        $output_file = "./RELEASE_NOTES.md"
+        $regex = "(?sm)^(## \d{1,2}\.\d{1,2}\.\d{1,2}(?:-[a-zA-Z]*\.\d{1,2})? \(\d{4}-\d{2}-\d{2}\))(.+?)(\n## \d{1,2}\.\d{1,2}\.\d{1,2}(?:-[a-zA-Z]*\.\d{1,2})? \(\d{4}-\d{2}-\d{2}\))"
+        $changelogContent -match $regex
+        Set-Content -Path $output_file -Value $Matches[2]
+        echo "::set-output name=release_notes::$output_file"
+    - name: Extract release name
+      id: extract_release_ver
+      run: |
+        $nupkgName = ls "Realm/packages" | Select -expandproperty Name | Select-String Realm.Fody
+        $nupkgName -match "(?sm)\d{1,2}\.\d{1,2}\.\d{1,2}(?:-[a-zA-Z]*\.\d{1,2})?"
+        $version = $Matches[0]
+        echo "::set-output name=release_version::$version"
+    - name: Upload to nuget
+      run: |
+        $release_version = ${{ steps.extract_release_ver.outputs.release_version }}
+        echo "Would upload to nuget 'Realm/packages/Realm.Fody.$release_version.nupkg/Realm.Fody.$release_version.nupkg'"
+        echo "Would upload to nuget 'Realm/packages/Realm.$release_version.nupkg/Realm.$release_version.nupkg'"
+    #     dotnet nuget push "Realm/packages/Realm.Fody.$release_version.nupkg/Realm.Fody.$release_version.nupkg" --skip-duplicate --api-key ${{ secrets.NUGET_API_KEY }} --source https://api.nuget.org/v3/index.json
+    #     dotnet nuget push "Realm/packages/Realm.$release_version.nupkg/Realm.$release_version.nupkg" --skip-duplicate --api-key ${{ secrets.NUGET_API_KEY }} --source https://api.nuget.org/v3/index.json
+    - name: Create Github Release
+      uses: ncipollo/release-action@v1
+      with:
+        artifacts: Realm/packages/io.realm.unity-${{ steps.extract_release_ver.outputs.release_version }}.tgz/io.realm.unity-${{ steps.extract_release_ver.outputs.release_version }}.tgz
+        bodyFile: ${{ steps.extract_release_notes.outputs.release_notes }}
+        name: "${{ steps.extract_release_ver.outputs.release_version }} - expand with the main features of this release"
+        commit: ${{ github.ref }}
+        tag: ${{ steps.extract_release_ver.outputs.release_version }}
+        token: ${{ secrets.GITHUB_TOKEN }}
+        draft: true
+    - name: Upload docs
+      run: |
+        Expand-Archive -Path Realm/packages/Docs.zip/Docs.zip -DestinationPath Realm/packages
+        py -m pip install s3cmd
+        $versions = "${{ steps.extract_release_ver.outputs.release_version }}", "latest"
+        Foreach ($ver in $versions)
+        {
+          s3cmd put --recursive --acl-public --access_key=${{ secrets.DOCS_S3_ACCESS_KEY }} --secret_key=${{ secrets.DOCS_S3_SECRET_KEY }} "${{ github.workspace }}/Realm/packages/_site s3://realm-sdks/realm-sdks/dotnet/$ver/
+        }