Overhaul the NuGet packaging infrastructure and the way the Core bina…

…ries get acquired. (#146) * Start writing an MSBuild script that creates NuGet packages for the core. For now it downloads the artifacts from GitHub Releases. * Support creating the native packages. * Support creating the metapackage. * (REMOVE BEFORE MERGE) Add a temporary MyGet feed for the TileDB native packages. * Use the TileDB Embedded page for the project URL in the native packages. * Use the `TileDB.Native` NuGet package and remove the nuspec of `TileDB.CSharp`. * Set `UseCurrentRuntimeIdentifier` in the executable projects. * Restore the template packages. And add a property to them that seems to save time. * Remove the ValueSeparator hack. * Remove the "Install/Download tiledb" steps from CI. And disable fail-fast in the Run-Tests action matrix. * Remove all CI jobs except of Run-Tests. Now that we don't use a nuspec to pack the C# library there are little reasons to specifically test it with a packed NuGet package. The Release job also went away; it would upload the 0.0.0-local package. We need a dedicated release workflow. * Remove RollForward from TileDB.CSharp. It does not apply to libraries. * Update package metadata and move them to a dedicated file. I had originally removed the nuspec files from the native packages but changed my mind for aesthetic reasons since they generate an empty .NET Standard 2.0 dependency group. * Add support for generating development editions of the native packages. They have SemVer 2.0 versions that identify the branch. * Enable Central Package Management with package source mapping. And support changing the native package under a different name in development builds. * Add release notes in the packages. * Refactor the nightly build workflow to use native development NuGet packages. The core is built in separate jobs and not once for each .NET version. * Update the repository README and add a package README. * Require `Version` and `VersionTag` to be specified. Make the version tag optional for development builds. And fail if no native binaries were found in development builds. * Add a README describing the native NuGet package generator. * Demand Core version 2.12.x. * Warn when building an RID-agnostic executable that uses `TileDB.CSharp`. * Remove the RID-specific build instructions in the metapackage's description. This package is not intended to be directly referenced by the average developer, at least not without `TileDB.CSharp`. * Remove the MyGet feed; we are now using the official TileDB binaries from NuGet. * Use MSBuild's custom item separator feature instead of a `string.Replace` hack.
TileDB-Inc · Dec 1, 2022 · 6f9fc19 · 6f9fc19
1 parent 5c217a9
commit 6f9fc19
Show file tree

Hide file tree

Showing 33 changed files with 464 additions and 920 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -5,7 +5,7 @@ indent_style = space
 indent_size = 4
 insert_final_newline = true
 
-[*.{xml,*proj,props,targets,yml}]
+[*.{xml,*proj,props,targets,yml,nuspec}]
 indent_size = 2
 
 [*.cs]

diff --git a/.github/scripts/download_tiledb.sh b/.github/scripts/download_tiledb.sh
diff --git a/.github/scripts/install_tiledb.sh b/.github/scripts/install_tiledb.sh
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -1,90 +1,135 @@
 name: Nightly-Build
 
+env:
+  TILEDB_NIGHTLY_BUILD: 1
+
 on:
   schedule:
     - cron: "15 2 * * *"
   workflow_dispatch:
 
 jobs:
-  Build:
+  Build-Native:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
         include:
-          - os: ubuntu-22.04
-            HOST: linux
-            BOOTSTRAP: ../bootstrap --enable-s3 --enable-serialization
+          - BOOTSTRAP: ../bootstrap --enable-s3 --enable-serialization
           - os: ubuntu-latest
-            HOST: linux
-            BOOTSTRAP: ../bootstrap --enable-s3 --enable-serialization
+            platform: linux-x86_64
           - os: macos-latest
-            HOST: osx
-            BOOTSTRAP: ../bootstrap --enable-s3 --enable-serialization
+            platform: macos-x86_64
           - os: windows-latest
-            HOST: win
+            platform: windows-x86_64
             BOOTSTRAP: ../bootstrap.ps1 -EnableS3 -EnableSerialization
         tag: [release-2.12, dev]
-        dotnet: ['net5.0', 'net6.0']
     runs-on: ${{ matrix.os }}
-
     steps:
-      - name: Checkout TileDB-CSharp
-        uses: actions/checkout@v3
-        with:
-          path: tiledb-csharp
-
       - name: Checkout TileDB
         uses: actions/checkout@v3
         with:
           repository: TileDB-Inc/TileDB
           ref: ${{ matrix.tag }}
-          path: tiledb
+
+      - name: Build TileDB
+        run: |
+          mkdir -p build
+          cd build
+          ${{ matrix.BOOTSTRAP }}
+          cmake --build . --config Release
+          cmake --build . --config Release --target install-tiledb
+          cd ../
+          mkdir -p artifacts/${{ matrix.platform }}
+          mv dist/* artifacts/${{ matrix.platform }}
+
+      # We combine the binaries of the many platforms per tag by uploading them to the same artifact.
+      - name: Upload native artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: tiledb-native-${{ matrix.tag }}
+          path: artifacts/
+
+  Pack-NuGet:
+    needs: Build-Native
+    strategy:
+      fail-fast: false
+      matrix:
+        tag: [release-2.12, dev]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout TileDB-CSharp
+        uses: actions/checkout@v3
 
       # GitHub runners come with several versions of .NET preinstalled; Remove them to target version
       - name: Remove existing .NET versions
         shell: bash
-        run: |
-          rm -rf $DOTNET_ROOT
+        run: rm -rf $DOTNET_ROOT
 
       - name: Set up .NET SDK from global.json
         uses: actions/setup-dotnet@v3
-        with:
-          global-json-file: tiledb-csharp/global.json
 
       - name: Display dotnet versions
         run: dotnet --info
 
-      - name: Build TileDB
-        run: |
-          mkdir -p tiledb/build
-          cd tiledb/build
-          ${{ matrix.BOOTSTRAP }}
-          cmake --build . --config Release
-          cmake --build . --config Release --target install-tiledb
-          cd ../../
-          cp -r tiledb/dist/lib/* tiledb-csharp/sources/TileDB.CSharp/runtimes/${{ matrix.HOST }}-x64/native/
+      - name: Download native artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: tiledb-native-${{ matrix.tag }}
+          path: scripts/nuget/temp
 
-      - name: Copy Windows tiledb.dll
-        if: matrix.HOST == 'win'
-        run: |
-          cp tiledb/dist/bin/tiledb.dll tiledb-csharp/sources/TileDB.CSharp/runtimes/${{ matrix.HOST }}-x64/native/
+      - name: Build native NuGet packages
+        shell: bash
+        run: dotnet pack ./scripts/nuget/GenerateNuGetPackages.proj -p:DevelopmentBuild=true -p:VersionTag=${{ matrix.tag }}
+
+      - name: Upload native NuGet packages
+        uses: actions/upload-artifact@v3
+        with:
+          name: tiledb-nuget-${{ matrix.tag }}
+          path: scripts/nuget/packages/
+
+  Test-NuGet:
+    needs: Pack-NuGet
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-22.04, ubuntu-latest, macos-latest, windows-latest]
+        tag: [release-2.12, dev]
+        dotnet: ['net5.0', 'net6.0']
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout TileDB-CSharp
+        uses: actions/checkout@v3
+
+      # GitHub runners come with several versions of .NET preinstalled; Remove them to target version
+      - name: Remove existing .NET versions
+        shell: bash
+        run: rm -rf $DOTNET_ROOT
+
+      - name: Set up .NET SDK from global.json
+        uses: actions/setup-dotnet@v3
+
+      - name: Display dotnet versions
+        run: dotnet --info
+
+      - name: Download native NuGet packages
+        uses: actions/download-artifact@v3
+        with:
+          name: tiledb-nuget-${{ matrix.tag }}
+          path: packages
 
       - name: Build TileDB-CSharp
-        run: |
-          dotnet build tiledb-csharp/sources/TileDB.CSharp/TileDB.CSharp.csproj /p:Platform=x64 -c Release
+        run: dotnet build sources/TileDB.CSharp/TileDB.CSharp.csproj -c Release
 
       - name: Test TileDB-CSharp
-        run: |
-          dotnet test tiledb-csharp/tests/TileDB.CSharp.Test/TileDB.CSharp.Test.csproj -c Release -f ${{ matrix.dotnet }}
+        run: dotnet test tests/TileDB.CSharp.Test/TileDB.CSharp.Test.csproj -c Release -f ${{ matrix.dotnet }}
 
       - name: Run examples
         shell: bash
-        run: |
-          find tiledb-csharp/examples/ -name *.csproj -execdir dotnet run \;
+        run: find examples/ -name *.csproj | xargs -I{} dotnet run --project {} -f ${{ matrix.dotnet }}
 
   Create-Issue:
-    needs: Build
+    needs: Test-NuGet
     if: failure()
     runs-on: ubuntu-latest
     steps:

diff --git a/.github/workflows/tiledb-csharp.yml b/.github/workflows/tiledb-csharp.yml
@@ -2,7 +2,6 @@ name: TileDB-CSharp
 
 on:
   push:
-    tags: [ '*' ]
     branches: [ main ]
   pull_request:
     branches: [ main ]
@@ -11,6 +10,7 @@ on:
 jobs:
   Run-Tests:
     strategy:
+      fail-fast: false
       matrix:
         # Will be checking following versions
         dotnet: ['net5.0', 'net6.0']
@@ -21,11 +21,6 @@ jobs:
       # Checks out repository
       - uses: actions/checkout@v3
 
-      # Install tiledb
-      - name: Install tiledb
-        shell: bash
-        run: ./.github/scripts/install_tiledb.sh
-
       - name: Remove existing .NET versions
         shell: bash
         run: |
@@ -40,7 +35,7 @@ jobs:
       # DotNet build
       - name: Dotnet build for TileDB.CSharp
         run: |
-          dotnet build /p:Platform=x64 -c Release sources/TileDB.CSharp
+          dotnet build -c Release sources/TileDB.CSharp
 
       # DotNet test
       - name: Test TileDB.CSharp
@@ -51,96 +46,3 @@ jobs:
         shell: bash
         run: |
           find examples/ -name *.csproj | xargs -I{} dotnet run --project {} -f ${{ matrix.dotnet }}
-
-  Stage-Release-Candidate:
-    needs: Run-Tests
-    runs-on: ubuntu-latest
-    steps:
-      # Checks out repository
-      - uses: actions/checkout@v3
-
-      - name: Set up .NET SDK from global.json
-        uses: actions/setup-dotnet@v3
-
-      - name: Display dotnet versions
-        run: dotnet --info
-
-      # Download tiledb
-      - name: Download tiledb
-        run: ./.github/scripts/download_tiledb.sh
-
-      # DotNet build
-      - name: Dotnet build for TileDB.CSharp
-        run: |
-          dotnet build /p:Platform=x64 -c Release sources/TileDB.CSharp
-
-      # DotNet pack
-      - name: Dotnet pack for TileDB.CSharp
-        run: |
-          dotnet pack ./sources/TileDB.CSharp/TileDB.CSharp.csproj -p:Version=0.0.0-local -c Release
-
-      - name: Archive nuget artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: TileDB NuGet Package
-          path: sources/TileDB.CSharp/lib/TileDB.CSharp.*.nupkg
-
-  Test-NuGet-Release:
-    needs: Stage-Release-Candidate
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        dotnet: ['net5.0', 'net6.0']
-    runs-on: ${{ matrix.os }}
-    steps:
-      - name: Checkout TileDB-CSharp repository
-        uses: actions/checkout@v3
-
-      - name: Download TileDB.CSharp NuGet artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: TileDB NuGet Package
-          path: packages
-
-      # GitHub runners come with several versions of .NET preinstalled; Remove them to target version
-      - name: Remove existing .NET versions
-        shell: bash
-        run: |
-          rm -rf $DOTNET_ROOT
-
-      - name: Set up .NET SDK from global.json
-        uses: actions/setup-dotnet@v3
-
-      - name: Display dotnet versions
-        run: dotnet --info
-
-      - name: Setup NuGet
-        uses: nuget/setup-nuget@v1
-
-      # Run tests using NuGet release candidate
-      - name: Test TileDB.CSharp
-        run: |
-          dotnet test -c Release tests/TileDB.CSharp.Test.NuGet -f ${{ matrix.dotnet }}
-
-  Release:
-    # Only run this job if a tag was provided
-    if: startsWith(github.ref, 'refs/tags/')
-    needs: Test-NuGet-Release
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/download-artifact@v3
-        with:
-          name: TileDB NuGet Package
-
-      - name: Create Release
-        id: create_release
-        uses: softprops/action-gh-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          files: ./*.nupkg
-          tag_name: ${{ github.ref_name }}
-          name: ${{ github.ref_name }}
-          body: ${{ steps.github_release.outputs.changelog }}
-          draft: false
-          prerelease: false
diff --git a/.gitignore b/.gitignore
@@ -44,8 +44,6 @@
 
 TileDB-CSharp.sln.DotSettings.user
 
-build
-
 TileDBConfig.cmake
 TileDbTargets*.cmake
 tiledb.pc
@@ -54,3 +52,7 @@ tiledb.pc
 
 .DS_Store
 include
+scripts/nuget/temp
+*.binlog
+*.nupkg
+packages
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -2,7 +2,4 @@
   <PropertyGroup>
     <LanguageVersion>latest</LanguageVersion>
   </PropertyGroup>
-  <ItemGroup>
-    <PackageReference Include="SonarAnalyzer.CSharp" Version="8.48.0.56517" PrivateAssets="all" />
-  </ItemGroup>
 </Project>