diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 470822cbbd..47908ea2b5 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -26,9 +26,9 @@ - + https://github.com/dotnet/arcade - 469dcc0cbcba5221727c8a5b9eec4a478e24a780 + 1aff4eb33aa7cbf26ccd9fc43c17cb609a14dad4 @@ -44,9 +44,9 @@ https://github.com/dotnet/symreader-converter c5ba7c88f92e2dde156c324a8c8edc04d9fa4fe0 - + https://github.com/dotnet/xliff-tasks - f35dcbb16190810eb88b103fb1f2cd48a3a76cf0 + a0391b3beff0696561189b5881f8af3b651d64ac diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 6a59f753b5..9caf9b021d 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -27,6 +27,7 @@ __AlpineArch=armv7 __FreeBSDArch=arm __FreeBSDMachineArch=armv7 __IllumosArch=arm7 +__HaikuArch=arm __QEMUArch=arm __UbuntuArch=armhf __UbuntuRepo="http://ports.ubuntu.com/" @@ -85,8 +86,12 @@ __IllumosPackages+=" mit-krb5" __IllumosPackages+=" openssl" __IllumosPackages+=" zlib" -__HaikuPackages="gmp" +__HaikuPackages="gcc_syslibs" +__HaikuPackages+=" gcc_syslibs_devel" +__HaikuPackages+=" gmp" __HaikuPackages+=" gmp_devel" +__HaikuPackages+=" icu66" +__HaikuPackages+=" icu66_devel" __HaikuPackages+=" krb5" __HaikuPackages+=" krb5_devel" __HaikuPackages+=" libiconv" @@ -95,6 +100,10 @@ __HaikuPackages+=" llvm12_libunwind" __HaikuPackages+=" llvm12_libunwind_devel" __HaikuPackages+=" mpfr" __HaikuPackages+=" mpfr_devel" +__HaikuPackages+=" openssl" +__HaikuPackages+=" openssl_devel" +__HaikuPackages+=" zlib" +__HaikuPackages+=" zlib_devel" # ML.NET dependencies __UbuntuPackages+=" libomp5" @@ -210,6 +219,7 @@ while :; do __FreeBSDArch=amd64 __FreeBSDMachineArch=amd64 __illumosArch=x86_64 + __HaikuArch=x86_64 __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" ;; x86) @@ -340,7 +350,6 @@ while :; do ;; haiku) __CodeName=haiku - __BuildArch=x64 __SkipUnmount=1 ;; --skipunmount) @@ -559,67 +568,54 @@ elif [[ "$__CodeName" == "illumos" ]]; then elif [[ "$__CodeName" == "haiku" ]]; then JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} - echo "Building Haiku sysroot for x86_64" + echo "Building Haiku sysroot for $__HaikuArch" mkdir -p "$__RootfsDir/tmp" - cd "$__RootfsDir/tmp" - git clone -b hrev56235 https://review.haiku-os.org/haiku - git clone -b btrev43195 https://review.haiku-os.org/buildtools - cd "$__RootfsDir/tmp/buildtools" && git checkout 7487388f5110021d400b9f3b88e1a7f310dc066d - - # Fetch some unmerged patches - cd "$__RootfsDir/tmp/haiku" - ## Add development build profile (slimmer than nightly) - git fetch origin refs/changes/64/4164/1 && git -c commit.gpgsign=false cherry-pick FETCH_HEAD - - # Build jam - cd "$__RootfsDir/tmp/buildtools/jam" - make - - # Configure cross tools - echo "Building cross-compiler" - mkdir -p "$__RootfsDir/generated" - cd "$__RootfsDir/generated" - "$__RootfsDir/tmp/haiku/configure" -j"$JOBS" --sysroot "$__RootfsDir" --cross-tools-source "$__RootfsDir/tmp/buildtools" --build-cross-tools x86_64 - - # Build Haiku packages - echo "Building Haiku" - echo 'HAIKU_BUILD_PROFILE = "development-raw" ;' > UserProfileConfig - "$__RootfsDir/tmp/buildtools/jam/jam0" -j"$JOBS" -q 'package' 'Haiku' - - BaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" - - # Download additional packages - echo "Downloading additional required packages" + pushd "$__RootfsDir/tmp" + + mkdir "$__RootfsDir/tmp/download" + + echo "Downloading Haiku package tool" + git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 $__RootfsDir/tmp/script + wget -O "$__RootfsDir/tmp/download/hosttools.zip" $($__RootfsDir/tmp/script/fetch.sh --hosttools) + unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin" + + DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" + HpkgBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current" + + # Download Haiku packages + echo "Downloading Haiku packages" read -ra array <<<"$__HaikuPackages" for package in "${array[@]}"; do echo "Downloading $package..." # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60 # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598 - hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_x86_64","versionType":"LATEST","naturalLanguageCode":"en"}' \ - --header='Content-Type:application/json' "$BaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" - wget -P "$__RootfsDir/generated/download" "$hpkgDownloadUrl" + hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ + --header='Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" + wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" + done + for package in haiku haiku_devel; do + echo "Downloading $package..." + hpkgVersion="$(wget -qO- $HpkgBaseUrl | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" + wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" done - # Setup the sysroot - echo "Setting up sysroot and extracting needed packages" + # Set up the sysroot + echo "Setting up sysroot and extracting required packages" mkdir -p "$__RootfsDir/boot/system" - for file in "$__RootfsDir/generated/objects/haiku/x86_64/packaging/packages/"*.hpkg; do - "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file" - done - for file in "$__RootfsDir/generated/download/"*.hpkg; do - "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file" + for file in "$__RootfsDir/tmp/download/"*.hpkg; do + echo "Extracting $file..." + LD_LIBRARY_PATH="$__RootfsDir/tmp/bin" "$__RootfsDir/tmp/bin/package" extract -C "$__RootfsDir/boot/system" "$file" done + # Download buildtools + echo "Downloading Haiku buildtools" + wget -O "$__RootfsDir/tmp/download/buildtools.zip" $($__RootfsDir/tmp/script/fetch.sh --buildtools --arch=$__HaikuArch) + unzip -o "$__RootfsDir/tmp/download/buildtools.zip" -d "$__RootfsDir" + # Cleaning up temporary files echo "Cleaning up temporary files" + popd rm -rf "$__RootfsDir/tmp" - for name in "$__RootfsDir/generated/"*; do - if [[ "$name" =~ "cross-tools-" ]]; then - : # Keep the cross-compiler - else - rm -rf "$name" - fi - done elif [[ -n "$__CodeName" ]]; then if [[ "$__SkipSigCheck" == "0" ]]; then diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 1c9d212d13..a88d643c8a 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -6,6 +6,7 @@ unset(FREEBSD) unset(ILLUMOS) unset(ANDROID) unset(TIZEN) +unset(HAIKU) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) @@ -16,6 +17,7 @@ elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) set(ILLUMOS 1) elseif(EXISTS ${CROSS_ROOTFS}/boot/system/develop/headers/config/HaikuConfig.h) set(CMAKE_SYSTEM_NAME Haiku) + set(HAIKU 1) else() set(CMAKE_SYSTEM_NAME Linux) set(LINUX 1) @@ -67,13 +69,25 @@ elseif(TARGET_ARCH_NAME STREQUAL "armv6") endif() elseif(TARGET_ARCH_NAME STREQUAL "ppc64le") set(CMAKE_SYSTEM_PROCESSOR ppc64le) - set(TOOLCHAIN "powerpc64le-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/powerpc64le-alpine-linux-musl) + set(TOOLCHAIN "powerpc64le-alpine-linux-musl") + else() + set(TOOLCHAIN "powerpc64le-linux-gnu") + endif() elseif(TARGET_ARCH_NAME STREQUAL "riscv64") set(CMAKE_SYSTEM_PROCESSOR riscv64) - set(TOOLCHAIN "riscv64-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/riscv64-alpine-linux-musl) + set(TOOLCHAIN "riscv64-alpine-linux-musl") + else() + set(TOOLCHAIN "riscv64-linux-gnu") + endif() elseif(TARGET_ARCH_NAME STREQUAL "s390x") set(CMAKE_SYSTEM_PROCESSOR s390x) - set(TOOLCHAIN "s390x-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/s390x-alpine-linux-musl) + set(TOOLCHAIN "s390x-alpine-linux-musl") + else() + set(TOOLCHAIN "s390x-linux-gnu") + endif() elseif(TARGET_ARCH_NAME STREQUAL "x64") set(CMAKE_SYSTEM_PROCESSOR x86_64) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/x86_64-alpine-linux-musl) @@ -88,11 +102,15 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64") elseif(ILLUMOS) set(TOOLCHAIN "x86_64-illumos") elseif(HAIKU) - set(TOOLCHAIN "x64_64-unknown-haiku") + set(TOOLCHAIN "x86_64-unknown-haiku") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) - set(TOOLCHAIN "i686-linux-gnu") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) + set(TOOLCHAIN "i586-alpine-linux-musl") + else() + set(TOOLCHAIN "i686-linux-gnu") + endif() if(TIZEN) set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0") endif() @@ -198,10 +216,8 @@ elseif(HAIKU) return() endif() - set(SEARCH_PATH "${CROSS_ROOTFS}/generated/cross-tools-x86_64/bin") - find_program(EXEC_LOCATION_${exec} - PATHS ${SEARCH_PATH} + PATHS "${CROSS_ROOTFS}/cross-tools-x86_64/bin" NAMES "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" "${TOOLSET_PREFIX}${exec}") @@ -266,8 +282,11 @@ elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) + add_toolchain_linker_flag("--target=${TOOLCHAIN}") + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}") + endif() add_toolchain_linker_flag(-m32) - if(TIZEN) add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") @@ -277,6 +296,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") elseif(ILLUMOS) add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/amd64/lib") +elseif(HAIKU) + add_toolchain_linker_flag("-lnetwork") + add_toolchain_linker_flag("-lroot") endif() # Specify compile options @@ -307,6 +329,9 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") add_compile_options(-mfloat-abi=softfp) endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) + add_compile_options(--target=${TOOLCHAIN}) + endif() add_compile_options(-m32) add_compile_options(-Wno-error=unused-command-line-argument) endif() diff --git a/eng/common/cross/x64/sources.list.bionic b/eng/common/cross/x64/sources.list.bionic new file mode 100644 index 0000000000..a71ccadcff --- /dev/null +++ b/eng/common/cross/x64/sources.list.bionic @@ -0,0 +1,11 @@ +deb http://archive.ubuntu.com/ubuntu/ bionic main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ bionic main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted +deb-src http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted + +deb http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index 7aee4213e1..517401b688 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -64,7 +64,7 @@ if [ -z "$CLR_CC" ]; then if [ -z "$majorVersion" ]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. if [ "$compiler" = "clang" ]; then versions="16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" - elif [ "$compiler" = "gcc" ]; then versions="12 11 10 9 8 7 6 5 4.9"; fi + elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi for version in $versions; do _major="${version%%.*}" diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index a7bbf0f1fe..44ad26abf5 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -158,7 +158,7 @@ jobs: - template: /eng/common/templates/steps/component-governance.yml parameters: ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/release'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: disableComponentGovernance: false ${{ else }}: disableComponentGovernance: true diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 8ad03be3ec..38cf94ff88 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -287,6 +287,25 @@ function InstallDotNet([string] $dotnetRoot, [string] $runtimeSourceFeedKey = '', [switch] $noPath) { + $dotnetVersionLabel = "'sdk v$version'" + + if ($runtime -ne '' -and $runtime -ne 'sdk') { + $runtimePath = $dotnetRoot + $runtimePath = $runtimePath + "\shared" + if ($runtime -eq "dotnet") { $runtimePath = $runtimePath + "\Microsoft.NETCore.App" } + if ($runtime -eq "aspnetcore") { $runtimePath = $runtimePath + "\Microsoft.AspNetCore.App" } + if ($runtime -eq "windowsdesktop") { $runtimePath = $runtimePath + "\Microsoft.WindowsDesktop.App" } + $runtimePath = $runtimePath + "\" + $version + + $dotnetVersionLabel = "runtime toolset '$runtime/$architecture v$version'" + + if (Test-Path $runtimePath) { + Write-Host " Runtime toolset '$runtime/$architecture v$version' already installed." + $installSuccess = $true + Exit + } + } + $installScript = GetDotNetInstallScript $dotnetRoot $installParameters = @{ Version = $version @@ -323,18 +342,18 @@ function InstallDotNet([string] $dotnetRoot, } else { $location = "public location"; } - Write-Host "Attempting to install dotnet from $location." + Write-Host " Attempting to install $dotnetVersionLabel from $location." try { & $installScript @variation $installSuccess = $true break } catch { - Write-Host "Failed to install dotnet from $location." + Write-Host " Failed to install $dotnetVersionLabel from $location." } } if (-not $installSuccess) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from any of the specified locations." + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install $dotnetVersionLabel from any of the specified locations." ExitWithExitCode 1 } } @@ -399,7 +418,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # Locate Visual Studio installation or download x-copy msbuild. $vsInfo = LocateVisualStudio $vsRequirements if ($vsInfo -ne $null) { - $vsInstallDir = $vsInfo.installationPath + # Ensure vsInstallDir has a trailing slash + $vsInstallDir = Join-Path $vsInfo.installationPath "\" $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion diff --git a/eng/common/tools.sh b/eng/common/tools.sh index cf9fb1ea2d..e8d4789433 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -184,6 +184,35 @@ function InstallDotNetSdk { function InstallDotNet { local root=$1 local version=$2 + local runtime=$4 + + local dotnetVersionLabel="'$runtime v$version'" + if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then + runtimePath="$root" + runtimePath="$runtimePath/shared" + case "$runtime" in + dotnet) + runtimePath="$runtimePath/Microsoft.NETCore.App" + ;; + aspnetcore) + runtimePath="$runtimePath/Microsoft.AspNetCore.App" + ;; + windowsdesktop) + runtimePath="$runtimePath/Microsoft.WindowsDesktop.App" + ;; + *) + ;; + esac + runtimePath="$runtimePath/$version" + + dotnetVersionLabel="runtime toolset '$runtime/$architecture v$version'" + + if [ -d "$runtimePath" ]; then + echo " Runtime toolset '$runtime/$architecture v$version' already installed." + local installSuccess=1 + return + fi + fi GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript @@ -228,17 +257,17 @@ function InstallDotNet { for variationName in "${variations[@]}"; do local name="$variationName[@]" local variation=("${!name}") - echo "Attempting to install dotnet from $variationName." + echo " Attempting to install $dotnetVersionLabel from $variationName." bash "$install_script" "${variation[@]}" && installSuccess=1 if [[ "$installSuccess" -eq 1 ]]; then break fi - echo "Failed to install dotnet from $variationName." + echo " Failed to install $dotnetVersionLabel from $variationName." done if [[ "$installSuccess" -eq 0 ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from any of the specified locations." + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install $dotnetVersionLabel from any of the specified locations." ExitWithExitCode 1 fi } diff --git a/global.json b/global.json index d9271e927d..4d8f7304b0 100644 --- a/global.json +++ b/global.json @@ -32,7 +32,7 @@ "dotnet": "8.0.100-preview.3.23178.7" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23226.4", + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23262.5", "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23226.4" } }